From f31744a6136a83416bfd1867dc4283252e526651 Mon Sep 17 00:00:00 2001 From: Dimitris Date: Thu, 25 Feb 2021 17:33:44 +0200 Subject: [PATCH] Fix issues with months --- .../repositories/UsageStatsRepository.java | 298 ++++++++---------- .../services/SushiLiteServiceImpl.java | 45 ++- 2 files changed, 160 insertions(+), 183 deletions(-) diff --git a/src/main/java/eu/dnetlib/usagestats/repositories/UsageStatsRepository.java b/src/main/java/eu/dnetlib/usagestats/repositories/UsageStatsRepository.java index daa585d..6c05319 100755 --- a/src/main/java/eu/dnetlib/usagestats/repositories/UsageStatsRepository.java +++ b/src/main/java/eu/dnetlib/usagestats/repositories/UsageStatsRepository.java @@ -658,76 +658,76 @@ public class UsageStatsRepository { return "-1"; } -// public void executeItem(List reportItems, String itemIdentifier, -// String repositoryIdentifier, String itemDataType, Date beginDate, -// Date endDate, String granularity) { -// String[] split = itemIdentifier.split(":"); -// switch (split[0].toLowerCase()) { -// case "oid": -// executeOid(reportItems, itemIdentifier.replaceFirst(split[0] + ":", ""), repositoryIdentifier, itemDataType, beginDate, endDate, granularity); + public void executeItemIR(List reportItems, String repositoryIdentifier, String itemIdentifier, Date beginDate, + Date endDate, List metricType, String itemDataType, String granularity) { + + String[] split = itemIdentifier.split(":"); + switch (split[0].toLowerCase()) { + case "oid": + executeOid(reportItems, repositoryIdentifier, itemIdentifier.replaceFirst(split[0] + ":", ""), beginDate, endDate, metricType, itemDataType, granularity); + break; + case "doi": + executeDoi(reportItems, repositoryIdentifier, itemIdentifier.replaceFirst(split[0] + ":", ""), beginDate, endDate, metricType, itemDataType, granularity); + break; +// case "openaire": +// executeOpenaire(reportItems, itemIdentifier.replaceFirst(split[0] + ":", ""), repositoryIdentifier, itemDataType, beginDate, endDate, granularity); // break; -// case "doi": -// executeDoi(reportItems, itemIdentifier.replaceFirst(split[0] + ":", ""), repositoryIdentifier, itemDataType, beginDate, endDate, granularity); -// break; -//// case "openaire": -//// executeOpenaire(reportItems, itemIdentifier.replaceFirst(split[0] + ":", ""), repositoryIdentifier, itemDataType, beginDate, endDate, granularity); -//// break; -// default: -// } -// } -// private void executeOid(List reportItems, String oid, -// String repositoryIdentifier, String itemDataType, Date beginDate, -// Date endDate, String granularity) { -// Connection connection = null; -// PreparedStatement st = null; -// ResultSet rs = null; -// try { -// connection = usageStatsDB.getConnection(); -// //st = connection.prepareStatement("SELECT DISTINCT roid.id FROM public.result_oids roid, public.downloads_stats s WHERE s.result_id=roid.id AND roid.orid=? UNION SELECT DISTINCT roid.id FROM public.result_oids roid, public.views_stats s WHERE s.result_id=roid.id AND roid.orid=?"); -// st = connection.prepareStatement("SELECT DISTINCT roid.id FROM " + statsDB + ".result_oids roid, " + usagestatsImpalaDB + ".usage_stats us WHERE us.result_id=roid.id AND roid.oid=?"); -// st.setString(1, oid); -// //st.setString(2, oid); -// -// rs = st.executeQuery(); -// -// while (rs.next()) { -// executeOpenaire(reportItems, rs.getString(1), repositoryIdentifier, itemDataType, beginDate, endDate, granularity); -// } -// connection.close(); -// } catch (Exception e) { -// log.error("Oid to OpenAIRE id failed: ", e); -// } finally { -// DbUtils.closeQuietly(rs); -// DbUtils.closeQuietly(st); -// DbUtils.closeQuietly(connection); -// } -// } -// private void executeDoi(List reportItems, String doi, -// String repositoryIdentifier, String itemDataType, Date beginDate, -// Date endDate, String granularity) { -// Connection connection = null; -// PreparedStatement st = null; -// ResultSet rs = null; -// try { -// connection = usageStatsDB.getConnection(); -// //st = connection.prepareStatement("SELECT DISTINCT poid.id FROM public.result_pids poid, public.downloads_stats s WHERE s.result_id=poid.id AND poid.type='doi' AND poid.pid=? UNION SELECT DISTINCT poid.id FROM public.result_pids poid, public.views_stats s WHERE s.result_id=poid.id AND poid.type='doi' AND poid.pid=?"); -// st = connection.prepareStatement("SELECT DISTINCT poid.id FROM " + statsDB + ".result_pids poid, " + usageStatsDB + ".usage_stats us WHERE us.result_id=poid.id AND poid.type='Digital Object Identifier' AND poid.pid=?"); -// st.setString(1, doi); -// //st.setString(2, doi); -// -// rs = st.executeQuery(); -// -// while (rs.next()) { -// executeOpenaire(reportItems, rs.getString(1), repositoryIdentifier, itemDataType, beginDate, endDate, granularity); -// } -// } catch (Exception e) { -// log.error("Doi to OpenAIRE id failed: ", e); -// } finally { -// DbUtils.closeQuietly(rs); -// DbUtils.closeQuietly(st); -// DbUtils.closeQuietly(connection); -// } -// } + default: + } + } + private void executeOid(List reportItems, String repositoryIdentifier, + String oid, Date beginDate, Date endDate, List metricType, String itemDataType, String granularity) { + Connection connection = null; + PreparedStatement st = null; + ResultSet rs = null; + try { + connection = usageStatsDB.getConnection(); + //st = connection.prepareStatement("SELECT DISTINCT roid.id FROM public.result_oids roid, public.downloads_stats s WHERE s.result_id=roid.id AND roid.orid=? UNION SELECT DISTINCT roid.id FROM public.result_oids roid, public.views_stats s WHERE s.result_id=roid.id AND roid.orid=?"); + st = connection.prepareStatement("SELECT DISTINCT roid.id FROM " + statsDB + ".result_oids roid, " + usagestatsImpalaDB + ".usage_stats us WHERE us.result_id=roid.id AND roid.oid=?"); + st.setString(1, oid); + //st.setString(2, oid); + + rs = st.executeQuery(); + + while (rs.next()) { + executeBatchItemsIR(reportItems, repositoryIdentifier, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + } + connection.close(); + } catch (Exception e) { + log.error("Oid to OpenAIRE id failed: ", e); + } finally { + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(st); + DbUtils.closeQuietly(connection); + } + } + + private void executeDoi(List reportItems, String repositoryIdentifier, + String doi, Date beginDate, Date endDate, List metricType, String itemDataType, String granularity) { + Connection connection = null; + PreparedStatement st = null; + ResultSet rs = null; + + try { + connection = usageStatsDB.getConnection(); + //st = connection.prepareStatement("SELECT DISTINCT poid.id FROM public.result_pids poid, public.downloads_stats s WHERE s.result_id=poid.id AND poid.type='doi' AND poid.pid=? UNION SELECT DISTINCT poid.id FROM public.result_pids poid, public.views_stats s WHERE s.result_id=poid.id AND poid.type='doi' AND poid.pid=?"); + st = connection.prepareStatement("SELECT DISTINCT poid.id FROM " + statsDB + ".result_pids poid, " + usagestatsImpalaDB + ".usage_stats us WHERE us.result_id=poid.id AND poid.type='Digital Object Identifier' AND poid.pid=?"); + st.setString(1, doi); + //st.setString(2, doi); + + rs = st.executeQuery(); + + while (rs.next()) { + executeBatchItemsIR(reportItems, repositoryIdentifier, rs.getString(1), beginDate, endDate, metricType, itemDataType, granularity); + } + } catch (Exception e) { + log.error("Doi to OpenAIRE id failed: ", e); + } finally { + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(st); + DbUtils.closeQuietly(connection); + } + } // // private void executeOpenaire(List reportItems, String openaire, // String repositoryIdentifier, String itemDataType, Date beginDate, @@ -1497,6 +1497,7 @@ public class UsageStatsRepository { // DbUtils.closeQuietly(connection); // } // } + public void executeBatchItemsPR(List reportItems, String repositoryIdentifier, Date beginDate, Date endDate, String metricType, String dataType, String granularity) { @@ -1980,7 +1981,7 @@ public class UsageStatsRepository { String[] typeIdentifierArray = identifiersAll[i].split("#-#"); reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(4), rs.getString(3))); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(4), rs.getString(3))); // if (rs.getString(5).indexOf("doi") >= 0) { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); // } else { @@ -1996,7 +1997,7 @@ public class UsageStatsRepository { String[] typeIdentifierArray = identifiersAll[i].split("#-#"); reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(4))); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), null, rs.getString(4))); // if (rs.getString(5).indexOf("doi") >= 0) { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); // } else { @@ -2012,7 +2013,7 @@ public class UsageStatsRepository { String[] typeIdentifierArray = identifiersAll[i].split("#-#"); reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(3), null)); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), null)); // if (rs.getString(5).indexOf("doi") >= 0) { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); // } else { @@ -2082,7 +2083,7 @@ public class UsageStatsRepository { reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(4), rs.getString(3))); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(4), rs.getString(3))); // if (rs.getString(5).indexOf("doi") >= 0) { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); @@ -2105,7 +2106,7 @@ public class UsageStatsRepository { // } else { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); // } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(4))); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), null, rs.getString(4))); reportItems.add(reportItem); metricFound = true; } @@ -2117,7 +2118,7 @@ public class UsageStatsRepository { reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(3), null)); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), null)); // if (rs.getString(5).indexOf("doi") >= 0) { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); // } else { @@ -2156,10 +2157,12 @@ public class UsageStatsRepository { if (granularity.equalsIgnoreCase("totals")) { if (dataType.equals("")) { - st = connection.prepareStatement("SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct ro.oid,'#!#') as oid " - + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs " - + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id " + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + "GROUP BY rc.type, rs.title, us.result_id, rs.title, rs.year order by rc.type ASC;"); st.setString(1, beginDateStr); @@ -2176,11 +2179,8 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); String[] identifiersAll = rs.getString(7).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), rs.getString(3))); @@ -2193,13 +2193,9 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations"); String[] identifiersAll = rs.getString(7).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(3))); reportItems.add(reportItem); } @@ -2210,30 +2206,30 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Requests"); String[] identifiersAll = rs.getString(7).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), null)); reportItems.add(reportItem); } metricFound = true; } } else { - st = connection.prepareStatement("SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct ro.oid,'#!#') as oid " - + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro " + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.type=? AND rc.id=us.result_id " - + "AND us.result_id=rs.id AND ro.id=us.result_id GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year order by rc.type ASC;"); + + "AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year order by rc.type ASC;"); st.setString(1, beginDateStr); st.setString(2, endDateStr); st.setString(3, repositoryIdentifier); st.setString(4, itemIdentifier); st.setString(5, dataType); + rs = st.executeQuery(); COUNTER_Item_Usage reportItem = null; @@ -2244,12 +2240,10 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); String[] identifiersAll = rs.getString(7).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), rs.getString(3))); reportItems.add(reportItem); } @@ -2261,11 +2255,8 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Investigations"); String[] identifiersAll = rs.getString(7).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(3))); reportItems.add(reportItem); @@ -2277,11 +2268,8 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), "Regular", "Total_Items_Requests"); String[] identifiersAll = rs.getString(7).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(2), null)); reportItems.add(reportItem); @@ -2292,11 +2280,15 @@ public class UsageStatsRepository { } else if (granularity.equalsIgnoreCase("monthly")) { if (dataType.equals("")) { - st = connection.prepareStatement("SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct ro.oid,'#!#') as oid " - + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro " - + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC;"); + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id " + + "AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC;"); st.setString(1, beginDateStr); st.setString(2, endDateStr); st.setString(3, repositoryIdentifier); @@ -2334,13 +2326,10 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); String[] identifiersAll = rs.getString(8).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(4), rs.getString(3))); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(4), rs.getString(3))); // if (rs.getString(5).indexOf("doi") >= 0) { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); // } else { @@ -2353,13 +2342,10 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations"); String[] identifiersAll = rs.getString(8).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(4))); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), null, rs.getString(4))); // if (rs.getString(5).indexOf("doi") >= 0) { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); // } else { @@ -2372,13 +2358,10 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Requests"); String[] identifiersAll = rs.getString(8).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(3), null)); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), null)); // if (rs.getString(5).indexOf("doi") >= 0) { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); // } else { @@ -2397,10 +2380,14 @@ public class UsageStatsRepository { //fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem); } } else { - st = connection.prepareStatement("SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " - + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct ro.oid,'#!#') as oid " - + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + statsDB + ".result_oids ro " - + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.type=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC;"); + st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) " + + "SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, " + + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid " + + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, " + + statsDB + ".result_oids ro, tpd " + + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.type=? AND rc.id=us.result_id " + + "AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id " + + "GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC;"); st.setString(1, beginDateStr); st.setString(2, endDateStr); st.setString(3, repositoryIdentifier); @@ -2441,13 +2428,11 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations|Total_Items_Requests"); String[] identifiersAll = rs.getString(8).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(4), rs.getString(3))); + + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(4), rs.getString(3))); // if (rs.getString(5).indexOf("doi") >= 0) { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); @@ -2461,11 +2446,8 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Investigations"); String[] identifiersAll = rs.getString(8).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } // if (rs.getString(5).indexOf("doi") >= 0) { @@ -2473,7 +2455,7 @@ public class UsageStatsRepository { // } else { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", rs.getString(5))); // } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), null, rs.getString(4))); + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), null, rs.getString(4))); reportItems.add(reportItem); metricFound = true; } @@ -2481,13 +2463,11 @@ public class UsageStatsRepository { reportItem = new COUNTER_Item_Usage(rs.getString(6), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(7)), "Regular", "Total_Items_Requests"); String[] identifiersAll = rs.getString(8).split("#!#"); for (int i = 0; i < identifiersAll.length; i++) { - if (identifiersAll[i].indexOf("10.") == 0) { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", identifiersAll[i])); - } else { - reportItem.addIdentifier(new COUNTER_Item_Identifiers("Proprietary", identifiersAll[i])); - } + String[] typeIdentifierArray = identifiersAll[i].split("#-#"); + reportItem.addIdentifier(new COUNTER_Item_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1])); } - reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), rs.getString(3), null)); + + reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(2))), report_dateFormat.format(endC.getTime()), rs.getString(3), null)); // if (rs.getString(5).indexOf("doi") >= 0) { // reportItem.addIdentifier(new COUNTER_Item_Identifiers("DOI", rs.getString(5))); // } else { diff --git a/src/main/java/eu/dnetlib/usagestats/services/SushiLiteServiceImpl.java b/src/main/java/eu/dnetlib/usagestats/services/SushiLiteServiceImpl.java index d4b15b6..4fecceb 100755 --- a/src/main/java/eu/dnetlib/usagestats/services/SushiLiteServiceImpl.java +++ b/src/main/java/eu/dnetlib/usagestats/services/SushiLiteServiceImpl.java @@ -452,7 +452,6 @@ public class SushiLiteServiceImpl implements SushiLiteService { reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); } - if (reportExceptions.size() == 0) { reportExceptions = null; } @@ -476,9 +475,9 @@ public class SushiLiteServiceImpl implements SushiLiteService { reportFilters.add(new Filter("EndDate", endDate)); repositoryIdentifier = usageStatsRepository.getInstitutionID(repositoryIdentifier); - + orgIdentifiers.add(new SUSHI_Org_Identifiers("Openaire", repositoryIdentifier)); - + if (!granularity.equalsIgnoreCase("totals") && !granularity.equalsIgnoreCase("monthly")) { reportExceptions.add(new SUSHI_Error_Model("3062", "Warning", "Invalid ReportAttribute Value", "usagecounts.openaire.eu", "Granularity: \'" + granularity + "\' unknown. Defaulting to Monthly")); granularity = "Monthly"; @@ -517,7 +516,6 @@ public class SushiLiteServiceImpl implements SushiLiteService { endDateParsed = temp.getTime(); reportExceptions.add(new SUSHI_Error_Model("3021", "Warning", "Unspecified Date Arguments", "usagecounts.openaire.eu", "End Date set to default: " + new SimpleDateFormat("yyyy-MM-dd").format(endDateParsed))); } - //log.error("dates: " + beginDateParsed.toString() + " - " + endDateParsed.toString()); if (beginDateParsed == null) { reportExceptions.add(new SUSHI_Error_Model("3020", "Error", "Invalid Date Arguments", "usagecounts.openaire.eu", "Begin Date: " + beginDate + " is not a valid date")); @@ -533,10 +531,10 @@ public class SushiLiteServiceImpl implements SushiLiteService { String institutionName = usageStatsRepository.getInstitutionName(repositoryIdentifier); //if (!repositoryIdentifier.equals("")) { - // repoid = usageStatsRepository.getInstitutionID(repositoryIdentifier); - if (repoid.equals("-1")) { - reportExceptions.add(new SUSHI_Error_Model("3060", "Error", "Invalid Filter Value", "usagecounts.openaire.eu", "RepositoryIdentifier: " + repositoryIdentifier + " is not valid")); - } + // repoid = usageStatsRepository.getInstitutionID(repositoryIdentifier); + if (repoid.equals("-1")) { + reportExceptions.add(new SUSHI_Error_Model("3060", "Error", "Invalid Filter Value", "usagecounts.openaire.eu", "RepositoryIdentifier: " + repositoryIdentifier + " is not valid")); + } //} String itemid = ""; if (!itemIdentifier.equals("")) { @@ -561,22 +559,23 @@ public class SushiLiteServiceImpl implements SushiLiteService { } //if (reportName.equalsIgnoreCase("ar1")) { if (!itemid.equals("-1") && !repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) { -// if (!itemid.equals("")) { -// if (dataType.equalsIgnoreCase("") || dataType.equalsIgnoreCase("article")) { -// usageStatsRepository.executeItem(reportItems, itemIdentifier, repoid, "Article", beginDateParsed, endDateParsed, granularity); -// if (reportItems.isEmpty()) { -// reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); -// } -// } else { -// reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); -// } -// } else if (!repoid.equals("")) { - usageStatsRepository.executeBatchItemsIR(reportItems, repoid, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); - + if (!itemid.equals("")) { + //if (dataType.equalsIgnoreCase("") || dataType.equalsIgnoreCase("article")) { + usageStatsRepository.executeItemIR(reportItems, repoid, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); + if (reportItems.isEmpty()) { + reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); + } + // } else { + //reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); + //} + } else if (!repoid.equals("")) { + usageStatsRepository.executeBatchItemsIR(reportItems, repoid, itemIdentifier, beginDateParsed, endDateParsed, metricType, dataType, granularity); + if (reportItems.isEmpty()) { reportExceptions.add(new SUSHI_Error_Model("3030", "Error", "usagecounts.openaire.eu", "No Usage Available for Requested Dates", "Service did not find any data")); } } + } // } // //} else if (reportName.equalsIgnoreCase("br1")) { // if (!itemid.equals("-1") && !repoid.equals("-1") && beginDateParsed != null && endDateParsed != null && beginDateParsed.before(endDateParsed)) { @@ -645,7 +644,7 @@ public class SushiLiteServiceImpl implements SushiLiteService { //} else { // reportExceptions.add(new SUSHI_Error_Model("3000", "Error", "usagecounts.openaire.eu", "Report " + reportName + " not supported", "Supported reports: AR1, IR1, RR1, BR1, BR2")); //} - + if (reportExceptions.size() == 0) { reportExceptions = null; } @@ -654,8 +653,7 @@ public class SushiLiteServiceImpl implements SushiLiteService { return reportResponse; } - - + // @Override // public COUNTER_Title_Report buildReportIR(String customerID, String repositoryIdentifier, String itemIdentifier, String beginDate, // String endDate, String metricType, String dataType, String granularity) { @@ -733,7 +731,6 @@ public class SushiLiteServiceImpl implements SushiLiteService { // COUNTER_Title_Report reportTR = new COUNTER_Title_Report(dateTime.format(formatter), customerID, reportID, reportName, institutionName, institutionIdD, reportExceptions, reportFilters, reportItems); // return reportTR; // } - @Override public String displayReportPR(String customerID, String repositoryIdentifier, String beginDate, String endDate, String metricType, String dataType, String granularity) {