2020-12-30 14:56:37 +01:00
package eu.dnetlib.usagestats.repositories ;
import org.apache.log4j.Logger ;
import org.springframework.data.redis.core.HashOperations ;
import org.springframework.data.redis.core.RedisTemplate ;
import org.springframework.stereotype.Repository ;
import com.fasterxml.jackson.databind.ObjectMapper ;
import eu.dnetlib.usagestats.portal.CountryRepositories ;
import eu.dnetlib.usagestats.portal.CountryUsageStats ;
import eu.dnetlib.usagestats.portal.CountryUsageStatsAll ;
import eu.dnetlib.usagestats.portal.MonthlyStats ;
import eu.dnetlib.usagestats.portal.MonthlyUsageStats ;
import eu.dnetlib.usagestats.portal.RepositoryStats ;
import eu.dnetlib.usagestats.portal.TotalStats ;
import eu.dnetlib.usagestats.portal.TotalStatsReposViewsDownloads ;
import eu.dnetlib.usagestats.portal.UsageStats ;
import eu.dnetlib.usagestats.portal.YearlyStats ;
2021-03-14 15:24:14 +01:00
import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Dataset_Identifiers ;
import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Dataset_Performance ;
import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Dataset_Usage ;
2021-02-25 09:10:47 +01:00
import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Item_Identifiers ;
2021-01-25 09:44:27 +01:00
import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Item_Performance ;
2021-02-22 18:39:27 +01:00
import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Item_Usage ;
2021-01-25 09:44:27 +01:00
import eu.dnetlib.usagestats.sushilite.domain.COUNTER_Platform_Usage ;
2021-03-14 15:24:14 +01:00
import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Consortium_Member_List ;
import eu.dnetlib.usagestats.sushilite.domain.SUSHI_Org_Identifiers ;
2020-12-30 14:56:37 +01:00
import org.apache.commons.dbutils.DbUtils ;
import javax.sql.DataSource ;
import java.security.MessageDigest ;
import java.sql.Connection ;
import java.sql.PreparedStatement ;
import java.sql.ResultSet ;
2021-02-25 09:10:47 +01:00
import java.sql.SQLException ;
2020-12-30 14:56:37 +01:00
import java.text.SimpleDateFormat ;
import java.util.ArrayList ;
import java.util.Calendar ;
import java.util.Date ;
import java.util.HashMap ;
import java.util.List ;
import org.springframework.beans.factory.annotation.Value ;
2021-03-14 15:24:14 +01:00
/ * *
* Created by D . Pierrakos
* /
2020-12-30 14:56:37 +01:00
@Repository
public class UsageStatsRepository {
private final DataSource usageStatsDB ;
private final HashOperations < String , String , String > jedis ;
private final Logger log = Logger . getLogger ( this . getClass ( ) ) ;
@Value ( " ${prod.statsdb} " )
private String statsDB ;
@Value ( " ${prod.usagestatsImpalaDB} " )
private String usagestatsImpalaDB ;
2021-01-05 13:22:12 +01:00
public UsageStatsRepository ( DataSource usageStatsDB ,
RedisTemplate < String , String > redisTemplate ) {
2020-12-30 14:56:37 +01:00
this . usageStatsDB = usageStatsDB ;
this . jedis = redisTemplate . opsForHash ( ) ;
}
private static String MD5 ( String string ) throws java . security . NoSuchAlgorithmException {
MessageDigest md = MessageDigest . getInstance ( " MD5 " ) ;
md . update ( string . getBytes ( ) ) ;
byte byteData [ ] = md . digest ( ) ;
StringBuilder sb = new StringBuilder ( ) ;
for ( byte aByteData : byteData ) {
sb . append ( Integer . toString ( ( aByteData & 0xff ) + 0x100 , 16 ) . substring ( 1 ) ) ;
}
return sb . toString ( ) ;
}
private static String toJson ( Object o ) throws com . fasterxml . jackson . core . JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper ( ) ;
return objectMapper . writeValueAsString ( o ) ;
}
private static UsageStats fromJson ( String string ) throws java . io . IOException {
ObjectMapper objectMapper = new ObjectMapper ( ) ;
return objectMapper . readValue ( string , UsageStats . class ) ;
}
/ *
private static List < ReportItem > reportItemsFromJson ( String string ) throws Exception {
ObjectMapper objectMapper = new ObjectMapper ( ) ;
2021-01-25 09:44:27 +01:00
return objectMapper . readValue ( string , objectMapper . getTypeFactory ( ) . constructCollectionType ( List . class , COUNTER_Platform_Usage . class ) ) ;
2020-12-30 14:56:37 +01:00
}
* /
public List < MonthlyUsageStats > executeMontlyUsageStats ( String query ) {
List < MonthlyUsageStats > montlhyList = new ArrayList < MonthlyUsageStats > ( ) ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
log . info ( connection . toString ( ) ) ;
st = connection . prepareStatement ( query ) ;
log . info ( st . toString ( ) ) ;
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
MonthlyUsageStats monthlyUsageStats = new MonthlyUsageStats ( ) ;
monthlyUsageStats . addDate ( rs . getString ( 1 ) ) ;
monthlyUsageStats . addDownloads ( rs . getString ( 2 ) ) ;
monthlyUsageStats . addViews ( rs . getString ( 3 ) ) ;
montlhyList . add ( monthlyUsageStats ) ;
}
} catch ( Exception e ) {
System . out . println ( e ) ;
}
try {
jedis . put ( " test " , " result " , toJson ( montlhyList ) ) ;
jedis . put ( " test " , " persistent " , " false " ) ;
jedis . put ( " test " , " fetchMode " , " 3 " ) ;
} catch ( Exception e ) {
System . out . println ( e ) ;
2021-01-05 13:22:12 +01:00
} finally {
2020-12-30 14:56:37 +01:00
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return montlhyList ;
}
2021-01-05 13:22:12 +01:00
public TotalStatsReposViewsDownloads executeTotalStatsReposViewsDownloads (
String query ) {
2020-12-30 14:56:37 +01:00
TotalStatsReposViewsDownloads totalStatsReposViewsDownlads = new TotalStatsReposViewsDownloads ( ) ;
String total_repos = " " ;
String views = " " ;
String downloads = " " ;
String redis_key = " " ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
log . info ( connection . toString ( ) ) ;
st = connection . prepareStatement ( query ) ;
log . info ( st . toString ( ) ) ;
rs = st . executeQuery ( ) ;
redis_key = MD5 ( st . toString ( ) ) ;
while ( rs . next ( ) ) {
totalStatsReposViewsDownlads . addRepositories ( rs . getString ( 1 ) ) ;
totalStatsReposViewsDownlads . addViews ( rs . getString ( 2 ) ) ;
totalStatsReposViewsDownlads . addDownloads ( rs . getString ( 3 ) ) ;
}
} catch ( Exception e ) {
System . out . println ( e ) ;
}
try {
jedis . put ( redis_key , " result " , toJson ( totalStatsReposViewsDownlads ) ) ;
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
} catch ( Exception e ) {
System . out . println ( e ) ;
2021-01-05 13:22:12 +01:00
} finally {
2020-12-30 14:56:37 +01:00
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return totalStatsReposViewsDownlads ;
}
public CountryUsageStatsAll executeCountryUsageStats ( String query ) {
CountryUsageStatsAll countryListAll = new CountryUsageStatsAll ( ) ;
List < CountryUsageStats > countryList = new ArrayList < CountryUsageStats > ( ) ;
String date = " " ;
String total_repos = " " ;
String views = " " ;
String downloads = " " ;
String redis_key = " redis_key " ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
int total_views = 0 ;
int total_downloads = 0 ;
try {
connection = usageStatsDB . getConnection ( ) ;
log . info ( connection . toString ( ) ) ;
st = connection . prepareStatement ( query ) ;
log . info ( st . toString ( ) ) ;
rs = st . executeQuery ( ) ;
redis_key = MD5 ( st . toString ( ) ) ;
while ( rs . next ( ) ) {
CountryUsageStats countryUsageStats = new CountryUsageStats ( ) ;
countryUsageStats . addCountry ( rs . getString ( 1 ) ) ;
countryUsageStats . addTotalRepos ( rs . getString ( 2 ) ) ;
countryUsageStats . addViews ( rs . getString ( 3 ) ) ;
countryUsageStats . addDownloads ( rs . getString ( 4 ) ) ;
total_views + = Integer . parseInt ( rs . getString ( 3 ) ) ;
total_downloads + = Integer . parseInt ( rs . getString ( 4 ) ) ;
countryList . add ( countryUsageStats ) ;
}
countryListAll . addViewsAll ( Integer . toString ( total_views ) ) ;
countryListAll . addDownloadsAll ( Integer . toString ( total_downloads ) ) ;
countryListAll . addCountryUsageStats ( countryList ) ;
} catch ( Exception e ) {
log . info ( e ) ;
System . out . println ( e ) ;
}
try {
jedis . put ( redis_key , " result " , toJson ( countryListAll ) ) ;
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
} catch ( Exception e ) {
System . out . println ( e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return countryListAll ;
}
2021-01-05 13:22:12 +01:00
public CountryUsageStats executeCountryUsageStats ( String query ,
String country ) {
2020-12-30 14:56:37 +01:00
CountryUsageStats countryUsageStats = new CountryUsageStats ( ) ;
String total_repos = " " ;
String views = " " ;
String downloads = " " ;
String redis_key = " " ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
int total_views = 0 ;
int total_downloads = 0 ;
try {
connection = usageStatsDB . getConnection ( ) ;
st = connection . prepareStatement ( query ) ;
2021-01-05 13:22:12 +01:00
2020-12-30 14:56:37 +01:00
redis_key = MD5 ( st . toString ( ) ) ;
//st.setString(1, country);
log . info ( st . toString ( ) ) ;
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
countryUsageStats . addCountry ( country ) ;
countryUsageStats . addTotalRepos ( rs . getString ( 1 ) ) ;
countryUsageStats . addViews ( rs . getString ( 2 ) ) ;
countryUsageStats . addDownloads ( rs . getString ( 3 ) ) ;
}
} catch ( Exception e ) {
log . info ( e ) ;
System . out . println ( e ) ;
}
try {
jedis . put ( redis_key , " result " , toJson ( countryUsageStats ) ) ;
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
} catch ( Exception e ) {
System . out . println ( e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return countryUsageStats ;
}
public List < CountryRepositories > executeCountryRepositories ( String query ) {
List < CountryRepositories > countryReposList = new ArrayList < CountryRepositories > ( ) ;
String country = " " ;
String repository = " " ;
String redis_key = " " ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
log . info ( connection . toString ( ) ) ;
st = connection . prepareStatement ( query ) ;
log . info ( st . toString ( ) ) ;
rs = st . executeQuery ( ) ;
redis_key = MD5 ( st . toString ( ) ) ;
while ( rs . next ( ) ) {
CountryRepositories countryRepository = new CountryRepositories ( ) ;
countryRepository . addCountry ( rs . getString ( 1 ) ) ;
countryRepository . addRepository ( rs . getString ( 2 ) ) ;
countryReposList . add ( countryRepository ) ;
}
} catch ( Exception e ) {
System . out . println ( e ) ;
}
try {
jedis . put ( redis_key , " result " , toJson ( countryReposList ) ) ;
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
} catch ( Exception e ) {
System . out . println ( e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return countryReposList ;
}
2021-01-05 13:22:12 +01:00
public List < MonthlyUsageStats > executeMontlyUsageStatsForRepo ( String query ,
String datasourceId ) {
2020-12-30 14:56:37 +01:00
List < MonthlyUsageStats > montlhyList = new ArrayList < MonthlyUsageStats > ( ) ;
String redis_key = " " ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
st = connection . prepareStatement ( query ) ;
redis_key = MD5 ( st . toString ( ) ) ;
st . setString ( 1 , datasourceId ) ;
log . info ( connection . toString ( ) ) ;
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
MonthlyUsageStats monthlyUsageStats = new MonthlyUsageStats ( ) ;
monthlyUsageStats . addDate ( rs . getString ( 1 ) ) ;
monthlyUsageStats . addDownloads ( rs . getString ( 2 ) ) ;
monthlyUsageStats . addViews ( rs . getString ( 3 ) ) ;
montlhyList . add ( monthlyUsageStats ) ;
}
} catch ( Exception e ) {
System . out . println ( e ) ;
}
try {
jedis . put ( redis_key , " result " , toJson ( montlhyList ) ) ;
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
} catch ( Exception e ) {
System . out . println ( e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return montlhyList ;
}
2021-01-05 13:22:12 +01:00
public UsageStats executeUsageStats ( String query , List < String > values ,
String type ) {
2020-12-30 14:56:37 +01:00
UsageStats usageStats = new UsageStats ( ) ;
int total_views = 0 ;
int total_downloads = 0 ;
int page_views = 0 ;
int openaire_downloads = 0 ;
int openaire_views = 0 ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
st = connection . prepareStatement ( query ) ;
int i = 1 ;
for ( String s : values ) {
st . setString ( i , s ) ;
i + + ;
}
String redis_key = MD5 ( st . toString ( ) ) ;
String redis_result = jedis . get ( redis_key , " result " ) ;
if ( redis_result ! = null ) {
return fromJson ( redis_result ) ;
}
rs = st . executeQuery ( ) ;
if ( type . equals ( " result " ) ) {
while ( rs . next ( ) ) {
if ( rs . getString ( 1 ) . equals ( " views " ) & & rs . getString ( 4 ) ! = null & & ! rs . getString ( 4 ) . equals ( " " ) & & ! rs . getString ( 4 ) . equals ( " null " ) ) {
usageStats . addViews ( new RepositoryStats ( rs . getString ( 3 ) , rs . getString ( 2 ) , rs . getString ( 4 ) , rs . getString ( 5 ) ) ) ;
total_views + = Integer . parseInt ( rs . getString ( 4 ) ) ;
openaire_views + = Integer . parseInt ( rs . getString ( 5 ) ) ;
} else if ( rs . getString ( 1 ) . equals ( " downloads " ) & & rs . getString ( 4 ) ! = null & & ! rs . getString ( 4 ) . equals ( " " ) & & ! rs . getString ( 4 ) . equals ( " null " ) ) {
usageStats . addDownloads ( new RepositoryStats ( rs . getString ( 3 ) , rs . getString ( 2 ) , rs . getString ( 4 ) , " 0 " ) ) ;
total_downloads + = Integer . parseInt ( rs . getString ( 4 ) ) ;
openaire_downloads + = Integer . parseInt ( rs . getString ( 5 ) ) ;
} else if ( rs . getString ( 1 ) . equals ( " pageviews " ) & & rs . getString ( 4 ) ! = null & & ! rs . getString ( 4 ) . equals ( " " ) & & ! rs . getString ( 4 ) . equals ( " null " ) ) {
page_views = Integer . parseInt ( rs . getString ( 4 ) ) ;
}
}
usageStats . setTotal_views ( Integer . toString ( total_views ) ) ;
usageStats . setTotal_downloads ( Integer . toString ( total_downloads ) ) ;
usageStats . setPageViews ( Integer . toString ( page_views ) ) ;
usageStats . setTotal_openaire_views ( Integer . toString ( openaire_views ) ) ;
usageStats . setTotal_openaire_downloads ( Integer . toString ( openaire_downloads ) ) ;
} else if ( type . equals ( " project " ) | | type . equals ( " datasource " ) ) {
while ( rs . next ( ) ) {
if ( rs . getString ( 1 ) . equals ( " views " ) & & rs . getString ( 2 ) ! = null & & ! rs . getString ( 2 ) . equals ( " " ) & & ! rs . getString ( 2 ) . equals ( " null " ) ) {
total_views + = Integer . parseInt ( rs . getString ( 2 ) ) ;
openaire_views + = Integer . parseInt ( rs . getString ( 3 ) ) ;
} else if ( rs . getString ( 1 ) . equals ( " downloads " ) & & rs . getString ( 2 ) ! = null & & ! rs . getString ( 2 ) . equals ( " " ) & & ! rs . getString ( 2 ) . equals ( " null " ) ) {
total_downloads + = Integer . parseInt ( rs . getString ( 2 ) ) ;
openaire_downloads + = Integer . parseInt ( rs . getString ( 3 ) ) ;
} else if ( rs . getString ( 1 ) . equals ( " pageviews " ) & & rs . getString ( 2 ) ! = null & & ! rs . getString ( 2 ) . equals ( " " ) & & ! rs . getString ( 2 ) . equals ( " null " ) ) {
page_views = Integer . parseInt ( rs . getString ( 2 ) ) ;
}
/ *
else if ( rs . getString ( 1 ) . equals ( " openaire " ) & & rs . getString ( 2 ) ! = null & & ! rs . getString ( 2 ) . equals ( " " ) & & ! rs . getString ( 2 ) . equals ( " null " ) ) {
openaire = Integer . parseInt ( rs . getString ( 2 ) ) ;
}
* /
}
usageStats . setTotal_views ( Integer . toString ( total_views ) ) ;
usageStats . setTotal_downloads ( Integer . toString ( total_downloads ) ) ;
usageStats . setPageViews ( Integer . toString ( page_views ) ) ;
usageStats . setTotal_openaire_views ( Integer . toString ( openaire_views ) ) ;
usageStats . setTotal_openaire_downloads ( Integer . toString ( openaire_downloads ) ) ;
}
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " query " , st . toString ( ) ) ;
//jedis.put(redis_key, "result", toString(usageStats));
jedis . put ( redis_key , " result " , toJson ( usageStats ) ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
} catch ( Exception e ) {
log . error ( " Cannot execute query2 : " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return usageStats ;
}
public TotalStats executeTotalStats ( ) {
TotalStats totalStats = null ;
try {
String redis_result = jedis . get ( " total_stats " , " result " ) ;
if ( redis_result ! = null ) {
totalStats = fromJsonTotalStats ( redis_result ) ;
} else {
return updateTotalStats ( ) ;
}
} catch ( Exception e ) {
log . error ( " Cannot execute totalStats : " , e ) ;
}
return totalStats ;
}
public TotalStats updateTotalStats ( ) {
TotalStats totalStats = new TotalStats ( ) ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
HashMap < Integer , List < MonthlyStats > > monthlyStatsMap = new HashMap < > ( ) ;
try {
connection = usageStatsDB . getConnection ( ) ;
//st = connection.prepareStatement("SELECT count(distinct d.repository_id) AS repository, count(distinct d.result_id) AS items, sum(d.count) AS downloads, sum(v.count) AS views from public.downloads_stats d FULL OUTER JOIN public.views_stats v ON d.source=v.source AND d.repository_id=v.repository_id AND d.result_id=v.result_id AND d.date=v.date;");
2021-01-05 13:22:12 +01:00
st = connection . prepareStatement ( " SELECT ndv(distinct repository_id) AS repository, ndv(distinct result_id) AS items, sum(downloads) AS downloads, sum(views) AS views FROM " + usagestatsImpalaDB + " .usage_stats; " ) ;
2020-12-30 14:56:37 +01:00
rs = st . executeQuery ( ) ;
rs . next ( ) ;
totalStats . setRepositories ( rs . getInt ( 1 ) ) ;
totalStats . setItems ( rs . getInt ( 2 ) ) ;
totalStats . setDownloads ( rs . getInt ( 3 ) ) ;
totalStats . setViews ( rs . getInt ( 4 ) ) ;
rs . close ( ) ;
st . close ( ) ;
//st = connection.prepareStatement("select coalesce(d.date,v.date) as month, count(distinct d.repository_id) as repository, count(distinct d.result_id) as items, sum(d.count) as downloads, sum(v.count) as views from public.downloads_stats d FULL OUTER JOIN public.views_stats v ON d.source=v.source AND d.repository_id=v.repository_id AND d.result_id=v.result_id AND d.date=v.date group by month order by month;");
2021-01-05 13:22:12 +01:00
st = connection . prepareStatement ( " SELECT `date`, ndv(distinct repository_id) AS repository, ndv(distinct result_id) AS items, sum(downloads) AS downloads, sum(views) AS views FROM " + usagestatsImpalaDB + " .usage_stats GROUP BY `date` ORDER BY `date`; " ) ;
2020-12-30 14:56:37 +01:00
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
int year = Integer . parseInt ( rs . getString ( 1 ) . substring ( 0 , 4 ) ) ;
int month = Integer . parseInt ( rs . getString ( 1 ) . substring ( 5 ) ) ;
MonthlyStats monthlyStats = new MonthlyStats ( ) ;
monthlyStats . setMonth ( month ) ;
monthlyStats . setRepositories ( rs . getInt ( 2 ) ) ;
monthlyStats . setItems ( rs . getInt ( 3 ) ) ;
monthlyStats . setDownloads ( rs . getInt ( 4 ) ) ;
monthlyStats . setViews ( rs . getInt ( 5 ) ) ;
if ( monthlyStatsMap . get ( year ) ! = null ) {
monthlyStatsMap . get ( year ) . add ( monthlyStats ) ;
} else {
List < MonthlyStats > newList = new ArrayList < > ( ) ;
newList . add ( monthlyStats ) ;
monthlyStatsMap . put ( year , newList ) ;
}
}
rs . close ( ) ;
st . close ( ) ;
//st = connection.prepareStatement("SELECT COALESCE(SUBSTRING(d.date FROM 1 FOR 4), SUBSTRING(v.date FROM 1 FOR 4)) AS year, COUNT(DISTINCT d.repository_id) AS repository, COUNT(DISTINCT d.result_id) AS items, SUM(d.count) AS downloads, SUM(v.count) AS views FROM public.downloads_stats d FULL OUTER JOIN public.views_stats v ON d.source=v.source AND d.repository_id=v.repository_id AND d.result_id=v.result_id AND d.date=v.date GROUP BY year ORDER BY year;");
2021-01-05 13:22:12 +01:00
st = connection . prepareStatement ( " SELECT SUBSTR(`date`,1,4) AS year, ndv(DISTINCT repository_id) AS repository, \ n "
+ " ndv(DISTINCT result_id) AS items, SUM(downloads) AS downloads, SUM(views) AS views \ n "
+ " FROM " + usagestatsImpalaDB + " .usage_stats GROUP BY year ORDER BY year; " ) ;
2020-12-30 14:56:37 +01:00
rs = st . executeQuery ( ) ;
List < YearlyStats > yearlyStatsList = new ArrayList < > ( ) ;
while ( rs . next ( ) ) {
YearlyStats yearlyStats = new YearlyStats ( ) ;
yearlyStats . setYear ( rs . getInt ( 1 ) ) ;
yearlyStats . setRepositories ( rs . getInt ( 2 ) ) ;
yearlyStats . setItems ( rs . getInt ( 3 ) ) ;
yearlyStats . setDownloads ( rs . getInt ( 4 ) ) ;
yearlyStats . setViews ( rs . getInt ( 5 ) ) ;
yearlyStats . setMonthlyStats ( monthlyStatsMap . get ( rs . getInt ( 1 ) ) ) ;
yearlyStatsList . add ( yearlyStats ) ;
}
totalStats . setYearlyStats ( yearlyStatsList ) ;
jedis . put ( " total_stats " , " result " , toJson ( totalStats ) ) ;
jedis . put ( " total_stats " , " persistent " , " false " ) ;
} catch ( Exception e ) {
log . error ( " Cannot execute totalStats : " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return totalStats ;
}
private static TotalStats fromJsonTotalStats ( String string ) throws java . io . IOException {
ObjectMapper objectMapper = new ObjectMapper ( ) ;
return objectMapper . readValue ( string , TotalStats . class ) ;
}
public String executeRepoId ( String repositoryIdentifier , String report ) {
PreparedStatement st = null ;
Connection connection = null ;
ResultSet rs = null ;
2021-01-05 13:22:12 +01:00
log . info ( " database " + statsDB ) ;
2020-12-30 14:56:37 +01:00
try {
connection = usageStatsDB . getConnection ( ) ;
String [ ] split = repositoryIdentifier . split ( " : " ) ;
String openaire_id = " -1 " ;
switch ( split [ 0 ] . toLowerCase ( ) ) {
case " openaire " :
if ( ! report . equals ( " jr1 " ) ) {
2021-01-05 13:22:12 +01:00
st = connection . prepareStatement ( " select id from " + statsDB + " .datasource where id=? " ) ;
2020-12-30 14:56:37 +01:00
st . setString ( 1 , repositoryIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) ) ;
} else {
2021-01-05 13:22:12 +01:00
st = connection . prepareStatement ( " select id from " + statsDB + " .datasource where id=? AND (type='Journal' OR type='Journal Aggregator/Publisher') " ) ;
2020-12-30 14:56:37 +01:00
st . setString ( 1 , repositoryIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) ) ;
}
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
openaire_id = rs . getString ( 1 ) ;
}
return openaire_id ;
case " opendoar " :
if ( ! report . equals ( " jr1 " ) ) {
2021-01-05 13:22:12 +01:00
st = connection . prepareStatement ( " select id from " + statsDB + " .datasource_oids where oid=? " ) ;
2020-12-30 14:56:37 +01:00
st . setString ( 1 , " opendoar____:: " + repositoryIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) ) ;
} else {
2021-01-05 13:22:12 +01:00
st = connection . prepareStatement ( " select distinct d.id from " + statsDB + " .datasource d, " + statsDB + " .datasource_oids di where di.oid=? and d.id=di.id and (type='Journal' OR type='Journal Aggregator/Publisher') " ) ;
2020-12-30 14:56:37 +01:00
st . setString ( 1 , " opendoar____:: " + repositoryIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) ) ;
}
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
openaire_id = rs . getString ( 1 ) ;
}
return openaire_id ;
case " issn " :
2021-01-05 13:22:12 +01:00
st = connection . prepareStatement ( " select distinct d.id from " + statsDB + " .datasource d, " + statsDB + " .datasource_oids di, " + statsDB + " .datasource_results dr where d.id=dr.id and di.oid like ? and d.id=di.id and (type='Journal' OR type='Journal Aggregator/Publisher') " ) ;
2020-12-30 14:56:37 +01:00
st . setString ( 1 , " % " + repositoryIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) + " % " ) ;
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
openaire_id = rs . getString ( 1 ) ;
}
return openaire_id ;
default :
return " -1 " ;
}
} catch ( Exception e ) {
log . error ( " Repository id failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return " -1 " ;
}
2021-01-06 11:34:30 +01:00
public String getInstitutionID ( String repositoryIdentifier ) {
PreparedStatement st = null ;
Connection connection = null ;
ResultSet rs = null ;
log . info ( " database " + statsDB ) ;
try {
connection = usageStatsDB . getConnection ( ) ;
String [ ] split = repositoryIdentifier . split ( " : " ) ;
String openaire_id = " -1 " ;
switch ( split [ 0 ] . toLowerCase ( ) ) {
case " openaire " :
st = connection . prepareStatement ( " select id from " + statsDB + " .datasource where id=? " ) ;
st . setString ( 1 , repositoryIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) ) ;
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
openaire_id = rs . getString ( 1 ) ;
}
return openaire_id ;
case " opendoar " :
st = connection . prepareStatement ( " select id from " + statsDB + " .datasource_oids where oid=? " ) ;
st . setString ( 1 , " opendoar____:: " + repositoryIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) ) ;
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
openaire_id = rs . getString ( 1 ) ;
}
return openaire_id ;
case " issn " :
st = connection . prepareStatement ( " select distinct d.id from " + statsDB + " .datasource d, " + statsDB + " .datasource_oids di, " + statsDB + " .datasource_results dr where d.id=dr.id and di.oid like ? and d.id=di.id and (type='Journal' OR type='Journal Aggregator/Publisher') " ) ;
st . setString ( 1 , " % " + repositoryIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) + " % " ) ;
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
openaire_id = rs . getString ( 1 ) ;
}
return openaire_id ;
default :
return " -1 " ;
}
} catch ( Exception e ) {
log . error ( " Repository id failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return " -1 " ;
}
2021-02-25 16:33:44 +01:00
public void executeItemIR ( List < COUNTER_Item_Usage > reportItems , String repositoryIdentifier , String itemIdentifier , Date beginDate ,
Date endDate , List < String > metricType , String itemDataType , String granularity ) {
String [ ] split = itemIdentifier . split ( " : " ) ;
switch ( split [ 0 ] . toLowerCase ( ) ) {
case " oid " :
executeOid ( reportItems , repositoryIdentifier , itemIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) , beginDate , endDate , metricType , itemDataType , granularity ) ;
break ;
case " doi " :
executeDoi ( reportItems , repositoryIdentifier , itemIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) , beginDate , endDate , metricType , itemDataType , granularity ) ;
break ;
// case "openaire":
// executeOpenaire(reportItems, itemIdentifier.replaceFirst(split[0] + ":", ""), repositoryIdentifier, itemDataType, beginDate, endDate, granularity);
2021-02-22 18:39:27 +01:00
// break;
2021-02-25 16:33:44 +01:00
default :
}
}
2021-03-14 15:24:14 +01:00
public void executeItemDSR ( List < COUNTER_Dataset_Usage > reportItems , String repositoryIdentifier , String itemIdentifier , Date beginDate ,
2021-03-16 06:56:18 +01:00
Date endDate , List < String > metricType , String granularity ) {
2021-03-14 15:24:14 +01:00
String [ ] split = itemIdentifier . split ( " : " ) ;
switch ( split [ 0 ] . toLowerCase ( ) ) {
case " oid " :
2021-03-16 06:56:18 +01:00
executeDSROid ( reportItems , repositoryIdentifier , itemIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) , beginDate , endDate , metricType , granularity ) ;
2021-03-14 15:24:14 +01:00
break ;
case " doi " :
2021-03-16 06:56:18 +01:00
executeDSRDoi ( reportItems , repositoryIdentifier , itemIdentifier . replaceFirst ( split [ 0 ] + " : " , " " ) , beginDate , endDate , metricType , granularity ) ;
2021-03-14 15:24:14 +01:00
break ;
// case "openaire":
// executeOpenaire(reportItems, itemIdentifier.replaceFirst(split[0] + ":", ""), repositoryIdentifier, itemDataType, beginDate, endDate, granularity);
// break;
default :
}
}
2021-02-25 16:33:44 +01:00
private void executeOid ( List < COUNTER_Item_Usage > reportItems , String repositoryIdentifier ,
String oid , Date beginDate , Date endDate , List < String > metricType , String itemDataType , String granularity ) {
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
//st = connection.prepareStatement("SELECT DISTINCT roid.id FROM public.result_oids roid, public.downloads_stats s WHERE s.result_id=roid.id AND roid.orid=? UNION SELECT DISTINCT roid.id FROM public.result_oids roid, public.views_stats s WHERE s.result_id=roid.id AND roid.orid=?");
st = connection . prepareStatement ( " SELECT DISTINCT roid.id FROM " + statsDB + " .result_oids roid, " + usagestatsImpalaDB + " .usage_stats us WHERE us.result_id=roid.id AND roid.oid=? " ) ;
st . setString ( 1 , oid ) ;
//st.setString(2, oid);
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
2021-03-14 15:24:14 +01:00
if ( repositoryIdentifier ! = null ) {
executeBatchItemsIR ( reportItems , repositoryIdentifier , rs . getString ( 1 ) , beginDate , endDate , metricType , itemDataType , granularity ) ;
} else {
executeItemsAllRepoIR ( reportItems , rs . getString ( 1 ) , beginDate , endDate , metricType , itemDataType , granularity ) ;
}
2021-02-25 16:33:44 +01:00
}
connection . close ( ) ;
} catch ( Exception e ) {
log . error ( " Oid to OpenAIRE id failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
}
private void executeDoi ( List < COUNTER_Item_Usage > reportItems , String repositoryIdentifier ,
String doi , Date beginDate , Date endDate , List < String > metricType , String itemDataType , String granularity ) {
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
//st = connection.prepareStatement("SELECT DISTINCT poid.id FROM public.result_pids poid, public.downloads_stats s WHERE s.result_id=poid.id AND poid.type='doi' AND poid.pid=? UNION SELECT DISTINCT poid.id FROM public.result_pids poid, public.views_stats s WHERE s.result_id=poid.id AND poid.type='doi' AND poid.pid=?");
st = connection . prepareStatement ( " SELECT DISTINCT poid.id FROM " + statsDB + " .result_pids poid, " + usagestatsImpalaDB + " .usage_stats us WHERE us.result_id=poid.id AND poid.type='Digital Object Identifier' AND poid.pid=? " ) ;
st . setString ( 1 , doi ) ;
//st.setString(2, doi);
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
2021-03-14 15:24:14 +01:00
if ( repositoryIdentifier ! = null ) {
executeBatchItemsIR ( reportItems , repositoryIdentifier , rs . getString ( 1 ) , beginDate , endDate , metricType , itemDataType , granularity ) ;
} else {
executeItemsAllRepoIR ( reportItems , rs . getString ( 1 ) , beginDate , endDate , metricType , itemDataType , granularity ) ;
}
}
} catch ( Exception e ) {
log . error ( " Doi to OpenAIRE id failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
}
private void executeDSROid ( List < COUNTER_Dataset_Usage > reportItems , String repositoryIdentifier ,
2021-03-16 06:56:18 +01:00
String oid , Date beginDate , Date endDate , List < String > metricType , String granularity ) {
2021-03-14 15:24:14 +01:00
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
//st = connection.prepareStatement("SELECT DISTINCT roid.id FROM public.result_oids roid, public.downloads_stats s WHERE s.result_id=roid.id AND roid.orid=? UNION SELECT DISTINCT roid.id FROM public.result_oids roid, public.views_stats s WHERE s.result_id=roid.id AND roid.orid=?");
st = connection . prepareStatement ( " SELECT DISTINCT roid.id FROM " + statsDB + " .result_oids roid, " + usagestatsImpalaDB + " .usage_stats us, "
+ statsDB + " .result_classifications rc "
+ " WHERE us.result_id=roid.id AND rc.id=us.result_id AND rc.type='Dataset' AND roid.oid=? " ) ;
st . setString ( 1 , oid ) ;
//st.setString(2, oid);
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
if ( repositoryIdentifier ! = null ) {
2021-03-16 06:56:18 +01:00
executeBatchItemsDSR ( reportItems , repositoryIdentifier , rs . getString ( 1 ) , beginDate , endDate , metricType , granularity ) ;
2021-03-14 15:24:14 +01:00
} else {
2021-03-16 06:56:18 +01:00
executeItemsAllRepoDSR ( reportItems , rs . getString ( 1 ) , beginDate , endDate , metricType , granularity ) ;
2021-03-14 15:24:14 +01:00
}
}
} catch ( Exception e ) {
log . error ( " Oid to OpenAIRE id failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
}
private void executeDSRDoi ( List < COUNTER_Dataset_Usage > reportItems , String repositoryIdentifier ,
2021-03-16 06:56:18 +01:00
String doi , Date beginDate , Date endDate , List < String > metricType , String granularity ) {
2021-03-14 15:24:14 +01:00
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
//st = connection.prepareStatement("SELECT DISTINCT poid.id FROM public.result_pids poid, public.downloads_stats s WHERE s.result_id=poid.id AND poid.type='doi' AND poid.pid=? UNION SELECT DISTINCT poid.id FROM public.result_pids poid, public.views_stats s WHERE s.result_id=poid.id AND poid.type='doi' AND poid.pid=?");
st = connection . prepareStatement ( " SELECT DISTINCT poid.id FROM " + statsDB + " .result_pids poid, " + usagestatsImpalaDB + " .usage_stats us, "
+ statsDB + " .result_classifications rc "
+ " WHERE us.result_id=poid.id AND poid.type='Digital Object Identifier' AND rc.id=us.result_id AND rc.type='Dataset' AND poid.pid=? " ) ;
st . setString ( 1 , doi ) ;
//st.setString(2, doi);
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
if ( repositoryIdentifier ! = null ) {
2021-03-16 06:56:18 +01:00
executeBatchItemsDSR ( reportItems , repositoryIdentifier , rs . getString ( 1 ) , beginDate , endDate , metricType , granularity ) ;
2021-03-14 15:24:14 +01:00
} else {
2021-03-16 06:56:18 +01:00
executeItemsAllRepoDSR ( reportItems , rs . getString ( 1 ) , beginDate , endDate , metricType , granularity ) ;
2021-03-14 15:24:14 +01:00
}
2021-02-25 16:33:44 +01:00
}
} catch ( Exception e ) {
log . error ( " Doi to OpenAIRE id failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
}
2021-02-22 18:39:27 +01:00
//
// private void executeOpenaire(List<COUNTER_Platform_Usage> reportItems, String openaire,
// String repositoryIdentifier, String itemDataType, Date beginDate,
// Date endDate, String granularity) {
// SimpleDateFormat report_dateFormat = new SimpleDateFormat("yyyy-MM-dd");
// SimpleDateFormat postgresFormat = new SimpleDateFormat("yyyy/MM");
// String beginDateStr = postgresFormat.format(beginDate);
// String endDateStr = postgresFormat.format(endDate);
//
// Connection connection = null;
// PreparedStatement st = null;
// ResultSet rs = null;
//
// /*
// Calendar startCalendar = Calendar.getInstance();
// startCalendar.setTime(beginDate);
// Calendar endCalendar = Calendar.getInstance();
// endCalendar.setTime(endDate);
// int diffYear = endCalendar.get(Calendar.YEAR) - startCalendar.get(Calendar.YEAR);
// int diffMonth = diffYear * 12 + endCalendar.get(Calendar.MONTH) - startCalendar.get(Calendar.MONTH);
// */
// try {
// connection = usageStatsDB.getConnection();
// if (repositoryIdentifier.equals("")) {
// if (itemDataType.equals("")) {
// //st = connection.prepareStatement("SELECT res.repository_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.ddate, oids.orid, res.downloads, res.views FROM (SELECT coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.result_id, vs.result_id) AS result_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.downloads_stats s WHERE s.date>=? AND s.date<=? AND s.result_id=? GROUP BY s.repository_id, s.result_id, s.date) AS ds FULL OUTER JOIN (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.views_stats s WHERE s.date>=? AND s.date<=? AND s.result_id=? GROUP BY s.repository_id, s.result_id, s.date) AS vs ON ds.result_id=vs.result_id AND ds.date=vs.date) AS res JOIN public.result r ON res.result_id=r.id JOIN public.datasource d ON d.id=res.repository_id JOIN public.result_classifications rc ON rc.id=r.id LEFT JOIN (SELECT pids.id, string_agg(pids.pid, '#!#') AS pid FROM public.result_pids pids WHERE pids.id=? AND type='doi' GROUP BY pids.id) AS pids ON pids.id=r.id LEFT JOIN (SELECT oids.id, string_agg(oids.orid, '#!#') AS orid FROM public.result_oids oids WHERE oids.id=? GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.repository_id, res.ddate;");
// //st = connection.prepareStatement("SELECT res.repository_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.date, oids.orid, res.downloads, res.views FROM (SELECT us.repository_id, us.result_id, us.date, us.downloads, us.views FROM usage_stats us WHERE us.date>=? AND us.date<=? AND us.result_id=?) AS res JOIN public.result r ON res.result_id=r.id JOIN public.datasource d ON d.id=res.repository_id JOIN public.result_classifications rc ON rc.id=r.id LEFT JOIN (SELECT pids.id, string_agg(pids.pid, '#!#') AS pid FROM public.result_pids pids WHERE pids.id=? AND type='doi' GROUP BY pids.id) AS pids ON pids.id=r.id LEFT JOIN (SELECT oids.id, string_agg(oids.orid, '#!#') AS orid FROM public.result_oids oids WHERE oids.id=? GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.repository_id, res.date;");
// st = connection.prepareStatement("SELECT distinct res.repository_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.`date`, oids.oid, res.downloads, res.views "
// + "FROM (SELECT us.repository_id, us.result_id, us.`date`, us.downloads, us.views FROM " + usagestatsImpalaDB + ".usage_stats us "
// + "WHERE us.`date`>=? AND us.`date`<=? AND us.result_id=?) AS res "
// + "JOIN " + statsDB + ".result r ON res.result_id=r.id "
// + "JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN (select id, group_concat(type,',') as type FROM " + statsDB + ".result_classifications where id=? "
// + "GROUP by id) rc ON rc.id=r.id "
// + "LEFT JOIN (SELECT pids.id, group_concat(pids.pid, '#!#') AS pid FROM " + statsDB + ".result_pids pids "
// + "WHERE pids.id=? AND type='Digital Object Identifier' GROUP BY pids.id) AS pids ON pids.id=r.id "
// + "LEFT JOIN (SELECT id, group_concat(oids.oid, '#!#') AS oid FROM " + statsDB + ".result_oids oids "
// + "WHERE oids.id=? GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.repository_id, res.`date`;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, openaire);
// //st.setString(4, beginDateStr);
// //st.setString(5, endDateStr);
// //st.setString(6, openaire);
// st.setString(4, openaire);
// st.setString(5, openaire);
// st.setString(6, openaire);
// } else {
// //st = connection.prepareStatement("SELECT res.repository_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.ddate, oids.orid, res.downloads, res.views FROM (SELECT coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.result_id, vs.result_id) AS result_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.downloads_stats s WHERE s.date>=? AND s.date<=? AND s.result_id=? GROUP BY s.repository_id, s.result_id, s.date) AS ds FULL OUTER JOIN (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.views_stats s WHERE s.date>=? AND s.date<=? AND s.result_id=? GROUP BY s.repository_id, s.result_id, s.date) AS vs ON ds.result_id=vs.result_id AND ds.date=vs.date) AS res JOIN public.result r ON res.result_id=r.id JOIN public.datasource d ON d.id=res.repository_id JOIN public.result_classifications rc ON rc.id=r.id AND rc.type=? LEFT JOIN (SELECT pids.id, string_agg(pids.pid, '#!#') AS pid FROM public.result_pids pids WHERE pids.id=? AND type='doi' GROUP BY pids.id) AS pids ON pids.id=r.id LEFT JOIN (SELECT oids.id, string_agg(oids.orid, '#!#') AS orid FROM public.result_oids oids WHERE oids.id=? GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.repository_id, res.ddate;");
// st = connection.prepareStatement("SELECT distinct res.repository_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.`date`, oids.oid, res.downloads, "
// + "res.views FROM (SELECT us.repository_id, us.result_id, us.`date`, us.downloads, us.views FROM " + usagestatsImpalaDB + ".usage_stats us "
// + "WHERE us.`date`>=? AND us.`date`<=? AND us.result_id=?) AS res "
// + "JOIN " + statsDB + ".result r ON res.result_id=r.id "
// + "JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".result_classifications rc ON rc.id=r.id AND rc.type=? "
// + "LEFT JOIN (SELECT pids.id, group_concat(pids.pid, '#!#') AS pid "
// + "FROM " + statsDB + ".result_pids pids WHERE pids.id=? "
// + "AND type='Digital Object Identifier' GROUP BY pids.id) AS pids ON pids.id=r.id "
// + "LEFT JOIN (SELECT oids.id, group_concat(oids.oid, '#!#') AS oid "
// + "FROM " + statsDB + ".result_oids oids WHERE oids.id=? "
// + "GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.repository_id, res.`date`;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, openaire);
// //st.setString(4, beginDateStr);
// //st.setString(5, endDateStr);
// //st.setString(6, openaire);
// st.setString(4, itemDataType);
// st.setString(5, openaire);
// st.setString(6, openaire);
// }
// } else {
// if (itemDataType.equals("")) {
// //st = connection.prepareStatement("SELECT res.repository_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.ddate, oids.orid, res.downloads, res.views FROM (SELECT coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.result_id, vs.result_id) AS result_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.downloads_stats s WHERE s.date>=? AND s.date<=? AND s.result_id=? AND s.repository_id=? GROUP BY s.repository_id, s.result_id, s.date) AS ds FULL OUTER JOIN (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.views_stats s WHERE s.date>=? AND s.date<=? AND s.result_id=? AND s.repository_id=? GROUP BY s.repository_id, s.result_id, s.date) AS vs ON ds.result_id=vs.result_id AND ds.date=vs.date) AS res JOIN public.result r ON res.result_id=r.id JOIN public.datasource d ON d.id=res.repository_id JOIN public.result_classifications rc ON rc.id=r.id LEFT JOIN (SELECT pids.id, string_agg(pids.pid, '#!#') AS pid FROM public.result_pids pids WHERE pids.id=? AND type='doi' GROUP BY pids.id) AS pids ON pids.id=r.id LEFT JOIN (SELECT oids.id, string_agg(oids.orid, '#!#') AS orid FROM public.result_oids oids WHERE oids.id=? GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.repository_id, res.ddate;");
// st = connection.prepareStatement("SELECT distinct res.repository_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.`date`, oids.oid, res.downloads, res.views "
// + "FROM (SELECT us.repository_id, us.result_id, us.`date`, us.downloads, us.views "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us WHERE us.`date`>=? AND us.`date`<=? AND us.result_id=? AND us.repository_id=?) AS res "
// + "JOIN " + statsDB + ".result r ON res.result_id=r.id JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN (select id, group_concat(type,',') as type from " + statsDB + ".result_classifications where id=? group by id) rc ON rc.id=r.id "
// + "LEFT JOIN (SELECT pids.id, group_concat(pids.pid, '#!#') AS pid FROM " + statsDB + ".result_pids pids "
// + "WHERE pids.id=? AND type='Digital Object Identifier' GROUP BY pids.id) AS pids ON pids.id=r.id "
// + "LEFT JOIN (SELECT oids.id, group_concat(oids.oid, '#!#') AS oid FROM " + statsDB + ".result_oids oids "
// + "WHERE oids.id=? GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.repository_id, res.`date`;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, openaire);
// st.setString(4, repositoryIdentifier);
// //st.setString(5, beginDateStr);
// //st.setString(6, endDateStr);
// //st.setString(7, openaire);
// //st.setString(8, repositoryIdentifier);
// st.setString(5, openaire);
// st.setString(6, openaire);
// st.setString(7, openaire);
// } else {
// //st = connection.prepareStatement("SELECT res.repository_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.ddate, oids.orid, res.downloads, res.views FROM (SELECT coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.result_id, vs.result_id) AS result_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.downloads_stats s WHERE s.date>=? AND s.date<=? AND s.result_id=? AND s.repository_id=? GROUP BY s.repository_id, s.result_id, s.date) AS ds FULL OUTER JOIN (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.views_stats s WHERE s.date>=? AND s.date<=? AND s.result_id=? AND s.repository_id=? GROUP BY s.repository_id, s.result_id, s.date) AS vs ON ds.result_id=vs.result_id AND ds.date=vs.date) AS res JOIN public.result r ON res.result_id=r.id JOIN public.datasource d ON d.id=res.repository_id JOIN public.result_classifications rc ON rc.id=r.id AND rc.type=? LEFT JOIN (SELECT pids.id, string_agg(pids.pid, '#!#') AS pid FROM public.result_pids pids WHERE pids.id=? AND type='doi' GROUP BY pids.id) AS pids ON pids.id=r.id LEFT JOIN (SELECT oids.id, string_agg(oids.orid, '#!#') AS orid FROM public.result_oids oids WHERE oids.id=? GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.repository_id, res.ddate;");
// st = connection.prepareStatement("SELECT distinct res.repository_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.`date`, oids.oid, res.downloads,res.views "
// + "FROM (SELECT us.repository_id, us.result_id, us.`date`, us.downloads, us.views FROM " + usagestatsImpalaDB + ".usage_stats us "
// + "WHERE us.`date`>=? AND us.`date`<=? AND us.result_id=? AND us.repository_id=?) AS res "
// + "JOIN " + statsDB + ".result r ON res.result_id=r.id JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".result_classifications rc ON rc.id=r.id AND rc.type=?' "
// + "LEFT JOIN (SELECT pids.id, group_concat(pids.pid, '#!#') AS pid FROM " + statsDB + ".result_pids pids "
// + "WHERE pids.id=? AND type='Digital Object Identifier' GROUP BY pids.id) AS pids ON pids.id=r.id "
// + "LEFT JOIN (SELECT oids.id, group_concat(oids.oid, '#!#') AS oid "
// + "FROM " + statsDB + ".result_oids oids WHERE oids.id=? "
// + "GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.repository_id, res.`date`;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, openaire);
// st.setString(4, repositoryIdentifier);
// //st.setString(5, beginDateStr);
// //st.setString(6, endDateStr);
// //st.setString(7, openaire);
// //st.setString(8, repositoryIdentifier);
// st.setString(5, itemDataType);
// st.setString(6, openaire);
// st.setString(7, openaire);
// }
// }
//
// rs = st.executeQuery();
// String repository = "";
// String lastDate = "";
// COUNTER_Platform_Usage reportItem = null;
// int ft_total = 0;
// int abstr = 0;
//
// if (granularity.equalsIgnoreCase("totals")) {
// while (rs.next()) {
// if (!rs.getString(1).equals(repository)) {
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), Integer.toString(ft_total), Integer.toString(abstr)));
// reportItems.add(reportItem);
// }
// repository = rs.getString(1);
// reportItem = new COUNTER_Platform_Usage(rs.getString(3), rs.getString(7), rs.getString(5), rs.getString(2), "");
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OpenAIRE", openaire));
// reportItem.addIdentifier(new COUNTER_Item_Identifier("URLs", rs.getString(4)));
// if (rs.getString(9) != null && !rs.getString(9).equals("")) {
// if (rs.getString(9).contains("#!#")) {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OAI", rs.getString(9).substring(0, rs.getString(9).indexOf("#!#"))));
// } else {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OAI", rs.getString(9)));
// }
// }
// if (rs.getString(6) != null && !rs.getString(6).equals("")) {
// if (rs.getString(6).contains("#!#")) {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("DOI", rs.getString(6).substring(0, rs.getString(6).indexOf("#!#"))));
// } else {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("DOI", rs.getString(6)));
// }
// }
// ft_total = 0;
// abstr = 0;
// }
// ft_total += rs.getInt(10);
// abstr += rs.getInt(11);
// }
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), Integer.toString(ft_total), Integer.toString(abstr)));
// reportItems.add(reportItem);
// }
// } else if (granularity.equalsIgnoreCase("monthly")) {
// Calendar endCal = Calendar.getInstance();
// endCal.setTime(postgresFormat.parse(endDateStr));
// endCal.add(Calendar.MONTH, 1);
// Date endDateForZeros = endCal.getTime();
// while (rs.next()) {
// if (!rs.getString(1).equals(repository)) {
// if (reportItem != null) {
// fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
// reportItems.add(reportItem);
// }
// repository = rs.getString(1);
// lastDate = beginDateStr;
// reportItem = new COUNTER_Platform_Usage(rs.getString(3), rs.getString(7), rs.getString(5), rs.getString(2), "");
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OpenAIRE", openaire));
// reportItem.addIdentifier(new COUNTER_Item_Identifier("URLs", rs.getString(4)));
// if (rs.getString(9) != null && !rs.getString(9).equals("")) {
// if (rs.getString(9).contains("#!#")) {
// String allOAIs = rs.getString(9);
// String[] oaiArray = allOAIs.split("#!#");
// for (int i = 0; i < oaiArray.length; i++) {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OAI", oaiArray[i]));
// }
// //reportItem.addIdentifier(new COUNTER_Item_Identifier("OAI", rs.getString(9).substring(0, rs.getString(9).indexOf("#!#"))));
//
// //reportItem.addIdentifier(new COUNTER_Item_Identifier("OAI", rs.getString(9).substring(0, rs.getString(9).indexOf("#!#"))));
// } else {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OAI", rs.getString(9)));
// }
// }
// if (rs.getString(6) != null && !rs.getString(6).equals("")) {
// if (rs.getString(6).contains("#!#")) {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("DOI", rs.getString(6).substring(0, rs.getString(6).indexOf("#!#"))));
// } else {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("DOI", rs.getString(6)));
// }
// }
// }
// fillWithZeros(postgresFormat.parse(lastDate), postgresFormat.parse(rs.getString(8)), reportItem);
// Calendar endC = Calendar.getInstance();
// endC.setTime(postgresFormat.parse(rs.getString(8)));
// endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE));
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(8))), report_dateFormat.format(endC.getTime()), rs.getString(10), rs.getString(11)));
// }
// endC.setTime(postgresFormat.parse(rs.getString(8)));
// endC.add(Calendar.MONTH, 1);
// lastDate = postgresFormat.format(endC.getTime());
// }
// if (reportItem != null) {
// fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
// reportItems.add(reportItem);
// }
// }
// } catch (Exception e) {
// log.error("Single Item Report failed: ", e);
// } finally {
// DbUtils.closeQuietly(rs);
// DbUtils.closeQuietly(st);
// DbUtils.closeQuietly(connection);
// }
// }
// public void executeRepo(List<COUNTER_Platform_Usage> reportItems,
// String repositoryIdentifier, String itemDataType, Date beginDate,
// Date endDate, String granularity) {
// SimpleDateFormat report_dateFormat = new SimpleDateFormat("yyyy-MM-dd");
// SimpleDateFormat postgresFormat = new SimpleDateFormat("yyyy/MM");
// String beginDateStr = postgresFormat.format(beginDate);
// String endDateStr = postgresFormat.format(endDate);
//
// Connection connection = null;
// PreparedStatement st = null;
// ResultSet rs = null;
//
// try {
// connection = usageStatsDB.getConnection();
//
// if (repositoryIdentifier.equals("")) {
// if (itemDataType.equals("")) {
// //st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.orid, res.ddate, res.downloads, res.views FROM (SELECT coalesce(ds.source, vs.source), coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.downloads_stats s WHERE s.date>=? AND s.date<=? GROUP BY s.source, s.repository_id, s.date) AS ds FULL OUTER JOIN (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.views_stats s WHERE s.date>=? AND s.date<=? GROUP BY s.source, s.repository_id, s.date) AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.date=vs.date) AS res JOIN public.datasource d ON d.id=res.repository_id JOIN public.datasource_oids dois ON d.id=dois.id WHERE dois.orid LIKE 'opendoar%' ORDER BY d.id, d.name, res.ddate ASC;");
// st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.oid, res.`date`, res.downloads, res.views "
// + "FROM (SELECT us.source, us.repository_id, us.`date`, sum(us.downloads) AS downloads, sum(us.views) AS views "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us WHERE us.`date`>=? AND us.`date`<=? GROUP BY us.source, us.repository_id, us.`date`) "
// + "AS res JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".datasource_oids dois ON d.id=dois.id WHERE dois.oid "
// + "LIKE 'opendoar%' ORDER BY d.id, d.name, res.`date` ASC;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// //st.setString(3, beginDateStr);
// //st.setString(4, endDateStr);
// } else {
// //st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.orid, res.ddate, res.downloads, res.views FROM (SELECT coalesce(ds.source, vs.source), coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.downloads_stats s, result_classifications rc WHERE rc.id=s.result_id AND s.date>=? AND s.date<=? AND rc.type=? GROUP BY s.source, s.repository_id, s.date) AS ds FULL OUTER JOIN (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.views_stats s, public.result_classifications rc WHERE rc.id=s.result_id AND s.date>=? AND s.date<=? AND rc.type=? GROUP BY s.source, s.repository_id, s.date) AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.date=vs.date) AS res JOIN public.datasource d ON d.id=res.repository_id JOIN public.datasource_oids dois ON d.id=dois.id WHERE dois.orid LIKE 'opendoar%' ORDER BY d.id, d.name, res.ddate ASC;");
// st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.oid, res.`date`, res.downloads, res.views "
// + "FROM (SELECT us.source, us.repository_id, us.`date`, sum(us.downloads) AS downloads, sum(us.views) AS views "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc "
// + "WHERE rc.id=us.result_id AND us.`date`>=? AND us.`date`<=? AND rc.type=? "
// + "GROUP BY us.source, us.repository_id, us.`date`) AS res "
// + "JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".datasource_oids dois ON d.id=dois.id WHERE dois.oid LIKE 'opendoar%' "
// + "ORDER BY d.id, d.name, res.`date` ASC;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, itemDataType);
// //st.setString(4, beginDateStr);
// //st.setString(5, endDateStr);
// //st.setString(6, itemDataType);
// }
// } else {
// if (itemDataType.equals("")) {
// //st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.orid, res.ddate, res.downloads, res.views FROM (SELECT coalesce(ds.source, vs.source), coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.downloads_stats s WHERE s.date>=? AND s.date<=? AND s.repository_id=? GROUP BY s.source, s.repository_id, s.date) AS ds FULL OUTER JOIN (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.views_stats s WHERE s.date>=? AND s.date<=? AND s.repository_id=? GROUP BY s.source, s.repository_id, s.date) AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.date=vs.date) AS res JOIN public.datasource d ON d.id=res.repository_id JOIN public.datasource_oids dois ON d.id=dois.id WHERE dois.orid LIKE 'opendoar%' ORDER BY d.id, d.name, res.ddate ASC;");
// st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.oid, res.`date`, res.downloads, res.views "
// + "FROM (SELECT us.source, us.repository_id, us.`date`, sum(us.downloads) AS downloads, sum(us.views) AS views "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? "
// + "GROUP BY us.source, us.repository_id, us.`date`) AS res "
// + "JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".datasource_oids dois ON d.id=dois.id "
// + "WHERE dois.oid LIKE 'opendoar%' ORDER BY d.id, d.name, res.`date` ASC;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, repositoryIdentifier);
// //st.setString(4, beginDateStr);
// //st.setString(5, endDateStr);
// //st.setString(6, repositoryIdentifier);
// } else {
// //st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.orid, res.ddate, res.downloads, res.views FROM (SELECT coalesce(ds.source, vs.source), coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.downloads_stats s, public.result_classifications rc WHERE rc.id=s.result_id AND s.date>=? AND s.date<=? AND rc.type=? AND s.repository_id=? GROUP BY s.source, s.repository_id, s.date) AS ds FULL OUTER JOIN (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.views_stats s, public.result_classifications rc WHERE rc.id=s.result_id AND s.date>=? AND s.date<=? AND rc.type=? AND s.repository_id=? GROUP BY s.source, s.repository_id, s.date) AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.date=vs.date) AS res JOIN public.datasource d ON d.id=res.repository_id JOIN public.datasource_oids dois ON d.id=dois.id WHERE dois.orid LIKE 'opendoar%' ORDER BY d.id, d.name, res.ddate ASC;");
// st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.oid, res.`date`, res.downloads, res.views "
// + "FROM (SELECT us.source, us.repository_id, us.`date`, sum(us.downloads) AS downloads, sum(us.views) AS views "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc "
// + "WHERE rc.id=us.result_id AND us.`date`>=? AND us.`date`<=? AND rc.type=? "
// + "AND us.repository_id=? GROUP BY us.source, us.repository_id, us.`date`) AS res "
// + "JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".datasource_oids dois ON d.id=dois.id "
// + "WHERE dois.oid LIKE 'opendoar%' ORDER BY d.id, d.name, res.`date` ASC;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, itemDataType);
// st.setString(4, repositoryIdentifier);
// //st.setString(5, beginDateStr);
// //st.setString(6, endDateStr);
// //st.setString(7, itemDataType);
// //st.setString(8, repositoryIdentifier);
// }
// }
// //log.error("RR STATEMENT: " + st);
//
// /*
// String redis_key = MD5(st.toString());
//
// if (jedis.hasKey(redis_key, "result")) {
// reportItems.addAll(reportItemsFromJson((String) jedis.get(redis_key, "result")));
// st.close();
// connection.close();
// return;
// }
// */
// rs = st.executeQuery();
// String repository = "";
// String lastDate = "";
// COUNTER_Platform_Usage reportItem = null;
//
// /*
// Calendar startCalendar = Calendar.getInstance();
// startCalendar.setTime(beginDate);
// Calendar endCalendar = Calendar.getInstance();
// endCalendar.setTime(endDate);
// */
// int ft_total = 0;
// int abstr = 0;
// if (granularity.equalsIgnoreCase("totals")) {
// while (rs.next()) {
// if (!rs.getString(1).equals(repository)) {
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), Integer.toString(ft_total), Integer.toString(abstr)));
// reportItems.add(reportItem);
// }
// repository = rs.getString(1);
// reportItem = new COUNTER_Platform_Usage(null, rs.getString(2), "Platform", null, "");
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OpenAIRE", rs.getString(1)));
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OpenDOAR", rs.getString(4).substring(rs.getString(4).lastIndexOf(":") + 1)));
// reportItem.addIdentifier(new COUNTER_Item_Identifier("URL", rs.getString(3)));
// ft_total = 0;
// abstr = 0;
// }
// ft_total += rs.getInt(6);
// abstr += rs.getInt(7);
// }
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), Integer.toString(ft_total), Integer.toString(abstr)));
// reportItems.add(reportItem);
// }
// } else if (granularity.equalsIgnoreCase("monthly")) {
// Calendar endCal = Calendar.getInstance();
// endCal.setTime(postgresFormat.parse(endDateStr));
// endCal.add(Calendar.MONTH, 1);
// Date endDateForZeros = endCal.getTime();
// while (rs.next()) {
// if (!rs.getString(1).equals(repository)) {
// if (reportItem != null) {
// fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
// reportItems.add(reportItem);
// }
// repository = rs.getString(1);
// lastDate = beginDateStr;
// reportItem = new COUNTER_Platform_Usage(null, rs.getString(2), "Platform", null, "");
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OpenAIRE", rs.getString(1)));
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OpenDOAR", rs.getString(4).substring(rs.getString(4).lastIndexOf(":") + 1)));
// reportItem.addIdentifier(new COUNTER_Item_Identifier("URL", rs.getString(3)));
// }
// fillWithZeros(postgresFormat.parse(lastDate), postgresFormat.parse(rs.getString(5)), reportItem);
// Calendar endC = Calendar.getInstance();
// endC.setTime(postgresFormat.parse(rs.getString(5)));
// endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE));
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(5))), report_dateFormat.format(endC.getTime()), rs.getString(6), rs.getString(7)));
// }
// endC.setTime(postgresFormat.parse(rs.getString(5)));
// endC.add(Calendar.MONTH, 1);
// lastDate = postgresFormat.format(endC.getTime());
// }
// if (reportItem != null) {
// fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
// reportItems.add(reportItem);
// }
// }
//
// /*
// jedis.put(redis_key, "persistent", "false");
// jedis.put(redis_key, "query", st.toString());
// jedis.put(redis_key, "result", toJson(reportItems));
// jedis.put(redis_key, "fetchMode", "3");
// */
// rs.close();
// st.close();
// connection.close();
// } catch (Exception e) {
// log.error("Repository Report failed: ", e);
// } finally {
// DbUtils.closeQuietly(rs);
// DbUtils.closeQuietly(st);
// DbUtils.closeQuietly(connection);
// }
// }
// public void executeJournal(List<COUNTER_Platform_Usage> reportItems,
// String repositoryIdentifier, String itemDataType, Date beginDate,
// Date endDate, String granularity) {
// SimpleDateFormat report_dateFormat = new SimpleDateFormat("yyyy-MM-dd");
// SimpleDateFormat postgresFormat = new SimpleDateFormat("yyyy/MM");
// String beginDateStr = postgresFormat.format(beginDate);
// String endDateStr = postgresFormat.format(endDate);
//
// Connection connection = null;
// PreparedStatement st = null;
// ResultSet rs = null;
//
// try {
// connection = usageStatsDB.getConnection();
//
// if (repositoryIdentifier.equals("")) {
// if (itemDataType.equals("")) {
// //st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.orid, res.ddate, res.downloads, res.views FROM (SELECT coalesce(ds.source, vs.source), coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.downloads_stats s WHERE s.date>=? AND s.date<=? GROUP BY s.source, s.repository_id, s.date) AS ds FULL OUTER JOIN (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.views_stats s WHERE s.date>=? AND s.date<=? GROUP BY s.source, s.repository_id, s.date) AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.date=vs.date) AS res JOIN public.datasource d ON d.id=res.repository_id JOIN public.datasource_oids dois ON d.id=dois.id WHERE (d.type='Journal' OR d.type='Journal Aggregator/Publisher') ORDER BY d.id, d.name, res.ddate ASC;");
// st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.oid, res.`date`, res.downloads, res.views "
// + "FROM (SELECT us.source, us.repository_id, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us WHERE us.`date`>=? AND us.`date`<=? GROUP BY us.source, us.repository_id, us.`date`) AS res "
// + "JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".datasource_oids dois ON d.id=dois.id "
// + "WHERE (d.type='Journal' OR d.type='Journal Aggregator/Publisher') ORDER BY d.id, d.name, res.`date` ASC;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// //st.setString(3, beginDateStr);
// //st.setString(4, endDateStr);
// } else {
// //st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.orid, res.ddate, res.downloads, res.views FROM (SELECT coalesce(ds.source, vs.source), coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.downloads_stats s, public.result_classifications rc WHERE rc.id=s.result_id AND s.date>=? AND s.date<=? AND rc.type=? GROUP BY s.source, s.repository_id, s.date) AS ds FULL OUTER JOIN (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.views_stats s, public.result_classifications rc WHERE rc.id=s.result_id AND s.date>=? AND s.date<=? AND rc.type=? GROUP BY s.source, s.repository_id, s.date) AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.date=vs.date) AS res JOIN public.datasource d ON d.id=res.repository_id JOIN public.datasource_oids dois ON d.id=dois.id WHERE (d.type='Journal' OR d.type='Journal Aggregator/Publisher') ORDER BY d.id, d.name, res.ddate ASC;");
// st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.oid, res.`date`, res.downloads, res.views "
// + "FROM (SELECT us.source, us.repository_id, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc "
// + "WHERE rc.id=us.result_id AND us.`date`>=? AND us.`date`<=? AND rc.type=? "
// + "GROUP BY us.source, us.repository_id, us.`date`) AS res "
// + "JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".datasource_oids dois ON d.id=dois.id "
// + "WHERE (d.type='Journal' OR d.type='Journal Aggregator/Publisher') ORDER BY d.id, d.name, res.`date` ASC;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, itemDataType);
// //st.setString(4, beginDateStr);
// //st.setString(5, endDateStr);
// //st.setString(6, itemDataType);
// }
// } else {
// if (itemDataType.equals("")) {
// //st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.orid, res.ddate, res.downloads, res.views FROM (SELECT coalesce(ds.source, vs.source), coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.downloads_stats s WHERE s.date>=? AND s.date<=? AND s.repository_id=? GROUP BY s.source, s.repository_id, s.date) AS ds FULL OUTER JOIN (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.views_stats s WHERE s.date>=? AND s.date<=? AND s.repository_id=? GROUP BY s.source, s.repository_id, s.date) AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.date=vs.date) AS res JOIN public.datasource d ON d.id=res.repository_id JOIN public.datasource_oids dois ON d.id=dois.id WHERE (d.type='Journal' OR d.type='Journal Aggregator/Publisher') ORDER BY d.id, d.name, res.ddate ASC;");
// st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.oid, res.`date`, res.downloads, res.views "
// + "FROM (SELECT us.source, us.repository_id, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? "
// + "GROUP BY us.source, us.repository_id, us.`date`) AS res JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".datasource_oids dois ON d.id=dois.id "
// + "WHERE (d.type='Journal' OR d.type='Journal Aggregator/Publisher') ORDER BY d.id, d.name, res.`date` ASC;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, repositoryIdentifier);
// //st.setString(4, beginDateStr);
// //st.setString(5, endDateStr);
// //st.setString(6, repositoryIdentifier);
// } else {
// //st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.orid, res.ddate, res.downloads, res.views FROM (SELECT coalesce(ds.source, vs.source), coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.downloads_stats s, public.result_classifications rc WHERE rc.id=s.result_id AND s.date>=? AND s.date<=? AND rc.type=? AND s.repository_id=? GROUP BY s.source, s.repository_id, s.date) AS ds FULL OUTER JOIN (SELECT s.source, s.repository_id, s.date, sum(s.count) FROM public.views_stats s, public.result_classifications rc WHERE rc.id=s.result_id AND s.date>=? AND s.date<=? AND rc.type=? AND s.repository_id=? GROUP BY s.source, s.repository_id, s.date) AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.date=vs.date) AS res JOIN public.datasource d ON d.id=res.repository_id JOIN public.datasource_oids dois ON d.id=dois.id WHERE (d.type='Journal' OR d.type='Journal Aggregator/Publisher') ORDER BY d.id, d.name, res.ddate ASC;");
// st = connection.prepareStatement("SELECT d.id, d.name, d.websiteurl, dois.oid, res.`date`, res.downloads, res.views "
// + "FROM (SELECT us.source, us.repository_id, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc WHERE rc.id=us.result_id "
// + "AND us.`date`>=?' AND us.`date`<=? AND rc.type=? AND us.repository_id=? "
// + "GROUP BY us.source, us.repository_id, us.`date`) AS res "
// + "JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".datasource_oids dois ON d.id=dois.id "
// + "WHERE (d.type='Journal' OR d.type='Journal Aggregator/Publisher') ORDER BY d.id, d.name, res.`date` ASC;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, itemDataType);
// st.setString(4, repositoryIdentifier);
// //st.setString(5, beginDateStr);
// //st.setString(6, endDateStr);
// //st.setString(7, itemDataType);
// //st.setString(8, repositoryIdentifier);
// }
// }
// //log.error("RR STATEMENT: " + st);
//
// /*
// String redis_key = MD5(st.toString());
//
// if (jedis.hasKey(redis_key, "result")) {
// reportItems.addAll(reportItemsFromJson((String) jedis.get(redis_key, "result")));
// st.close();
// connection.close();
// return;
// }
// */
// rs = st.executeQuery();
// String repository = "";
// String lastDate = "";
// COUNTER_Platform_Usage reportItem = null;
//
// /*
// Calendar startCalendar = Calendar.getInstance();
// startCalendar.setTime(beginDate);
// Calendar endCalendar = Calendar.getInstance();
// endCalendar.setTime(endDate);
// */
// int ft_total = 0;
// int abstr = 0;
// if (granularity.equalsIgnoreCase("totals")) {
// while (rs.next()) {
// if (!rs.getString(1).equals(repository)) {
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), Integer.toString(ft_total), Integer.toString(abstr)));
// reportItems.add(reportItem);
// }
// repository = rs.getString(1);
// reportItem = new COUNTER_Platform_Usage(null, rs.getString(2), "Platform", null, "");
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OpenAIRE", rs.getString(1)));
// reportItem.addIdentifier(new COUNTER_Item_Identifier("ISSN", rs.getString(4).substring(rs.getString(4).lastIndexOf(":") + 1)));
// if (rs.getString(3) != null) {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("URL", rs.getString(3)));
// }
// ft_total = 0;
// abstr = 0;
// }
// ft_total += rs.getInt(6);
// abstr += rs.getInt(7);
// }
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), Integer.toString(ft_total), Integer.toString(abstr)));
// reportItems.add(reportItem);
// }
// } else if (granularity.equalsIgnoreCase("monthly")) {
// Calendar endCal = Calendar.getInstance();
// endCal.setTime(postgresFormat.parse(endDateStr));
// endCal.add(Calendar.MONTH, 1);
// Date endDateForZeros = endCal.getTime();
// while (rs.next()) {
// if (!rs.getString(1).equals(repository)) {
// if (reportItem != null) {
// fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
// reportItems.add(reportItem);
// }
// repository = rs.getString(1);
// lastDate = beginDateStr;
// reportItem = new COUNTER_Platform_Usage(null, rs.getString(2), "Platform", null, "");
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OpenAIRE", rs.getString(1)));
// reportItem.addIdentifier(new COUNTER_Item_Identifier("ISSN", rs.getString(4).substring(rs.getString(4).lastIndexOf(":") + 1)));
// if (rs.getString(3) != null) {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("URL", rs.getString(3)));
// }
// }
// fillWithZeros(postgresFormat.parse(lastDate), postgresFormat.parse(rs.getString(5)), reportItem);
// Calendar endC = Calendar.getInstance();
// endC.setTime(postgresFormat.parse(rs.getString(5)));
// endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE));
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(5))), report_dateFormat.format(endC.getTime()), rs.getString(6), rs.getString(7)));
// }
// endC.setTime(postgresFormat.parse(rs.getString(5)));
// endC.add(Calendar.MONTH, 1);
// lastDate = postgresFormat.format(endC.getTime());
// }
// if (reportItem != null) {
// fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
// reportItems.add(reportItem);
// }
// }
//
// /*
// jedis.put(redis_key, "persistent", "false");
// jedis.put(redis_key, "query", st.toString());
// jedis.put(redis_key, "result", toJson(reportItems));
// jedis.put(redis_key, "fetchMode", "3");
// */
// rs.close();
// st.close();
// connection.close();
// } catch (Exception e) {
// log.error("Repository Report failed: ", e);
// } finally {
// DbUtils.closeQuietly(rs);
// DbUtils.closeQuietly(st);
// DbUtils.closeQuietly(connection);
// }
// }
// public void executeBatchItems(List<COUNTER_Platform_Usage> reportItems,
// String repositoryIdentifier, String itemDataType, Date beginDate,
// Date endDate, String granularity) {
// SimpleDateFormat report_dateFormat = new SimpleDateFormat("yyyy-MM-dd");
// SimpleDateFormat postgresFormat = new SimpleDateFormat("yyyy/MM");
// String beginDateStr = postgresFormat.format(beginDate);
// String endDateStr = postgresFormat.format(endDate);
//
// Connection connection = null;
// PreparedStatement st = null;
// ResultSet rs = null;
//
// try {
// connection = usageStatsDB.getConnection();
//
// if (itemDataType.equals("")) {
// //st = connection.prepareStatement("SELECT res.result_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.ddate, oids.orid, res.downloads, res.views FROM (SELECT coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.result_id, vs.result_id) AS result_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.downloads_stats s WHERE s.date>=? AND s.date<=? AND s.repository_id=? GROUP BY s.repository_id, s.result_id, s.date) AS ds FULL OUTER JOIN (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.views_stats s WHERE s.date>=? AND s.date<=? AND s.repository_id=? GROUP BY s.repository_id, s.result_id, s.date) AS vs ON ds.result_id=vs.result_id AND ds.date=vs.date) AS res JOIN public.result r ON res.result_id=r.id JOIN public.datasource d ON d.id=res.repository_id JOIN public.result_classifications rc ON rc.id=r.id LEFT JOIN (SELECT pids.id, string_agg(pids.pid, '#!#') AS pid FROM public.result_pids pids, public.result_datasources rd WHERE rd.id=pids.id AND type='doi' AND rd.datasource=? GROUP BY pids.id) AS pids ON pids.id=r.id LEFT JOIN (SELECT oids.id, string_agg(oids.orid, '#!#') AS orid FROM public.result_oids oids, public.result_datasources rd WHERE rd.id=oids.id AND rd.datasource=? GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.result_id, res.ddate;");
// //st = connection.prepareStatement("SELECT res.result_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.date, oids.orid, res.downloads, res.views FROM (SELECT us.repository_id, us.result_id, us.date, sum(us.downloads) as downloads, sum(us.views) as views FROM public.usage_stats us WHERE us.date>=? AND us.date<=? AND us.repository_id=? GROUP BY us.repository_id, us.result_id, us.date) AS res JOIN public.result r ON res.result_id=r.id JOIN public.datasource d ON d.id=res.repository_id JOIN public.result_classifications rc ON rc.id=r.id LEFT JOIN (SELECT pids.id, string_agg(pids.pid, '#!#') AS pid FROM public.result_pids pids, public.result_datasources rd WHERE rd.id=pids.id AND type='doi' AND rd.datasource=? GROUP BY pids.id) AS pids ON pids.id=r.id LEFT JOIN (SELECT oids.id, string_agg(oids.orid, '#!#') AS orid FROM public.result_oids oids, public.result_datasources rd WHERE rd.id=oids.id AND rd.datasource=? GROUP BY oids.id) AS oids ON oids.id=r.id ORDER BY res.result_id, res.date;");
// st = connection.prepareStatement("SELECT distinct res.result_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.`date`, oids.oid, res.downloads, res.views "
// + "FROM (SELECT us.repository_id, us.result_id, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views FROM " + usagestatsImpalaDB + ".usage_stats us "
// + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? "
// + "GROUP BY us.repository_id, us.result_id, us.`date`) AS res JOIN " + statsDB + ".result r ON res.result_id=r.id "
// + "JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".result_classifications rc ON rc.id=r.id "
// + "LEFT JOIN (SELECT pids.id, group_concat(pids.pid, '#!#') AS pid "
// + "FROM " + statsDB + ".result_pids pids, " + statsDB + ".result_datasources rd "
// + "WHERE rd.id=pids.id AND type='Digital Object Identifier' AND rd.datasource=? "
// + "GROUP BY pids.id) AS pids ON pids.id=r.id "
// + "LEFT JOIN (SELECT oids.id, group_concat(oids.oid, '#!#') AS oid "
// + "FROM " + statsDB + ".result_oids oids, " + statsDB + ".result_datasources rd "
// + "WHERE rd.id=oids.id AND rd.datasource=? GROUP BY oids.id) "
// + "AS oids ON oids.id=r.id ORDER BY res.result_id, res.`date`;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, repositoryIdentifier);
// //st.setString(4, beginDateStr);
// //st.setString(5, endDateStr);
// //st.setString(6, repositoryIdentifier);
// st.setString(4, repositoryIdentifier);
// st.setString(5, repositoryIdentifier);
// } else {
// //st = connection.prepareStatement("SELECT res.result_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.ddate, oids.orid, res.downloads, res.views FROM (SELECT coalesce(ds.repository_id, vs.repository_id) AS repository_id, coalesce(ds.result_id, vs.result_id) AS result_id, coalesce(ds.date, vs.date) AS ddate, coalesce(ds.sum, 0) AS downloads, coalesce(vs.sum,0) AS views FROM (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.downloads_stats s WHERE s.date>=? AND s.date<=? AND s.repository_id=? GROUP BY s.repository_id, s.result_id, s.date) AS ds FULL OUTER JOIN (SELECT s.repository_id, s.result_id, s.date, sum(s.count) FROM public.views_stats s WHERE s.date>=? AND s.date<=? AND s.repository_id=? GROUP BY s.repository_id, s.result_id, s.date) AS vs ON ds.result_id=vs.result_id AND ds.date=vs.date) AS res JOIN public.result r ON res.result_id=r.id JOIN public.datasource d ON d.id=res.repository_id JOIN public.result_classifications rc ON rc.id=r.id LEFT JOIN (SELECT pids.id, string_agg(pids.pid, '#!#') AS pid FROM public.result_pids pids, result_datasources rd WHERE rd.id=pids.id AND type='doi' AND rd.datasource=? GROUP BY pids.id) AS pids ON pids.id=r.id LEFT JOIN (SELECT oids.id, string_agg(oids.orid, '#!#') AS orid FROM public.result_oids oids, public.result_datasources rd WHERE rd.id=oids.id AND rd.datasource=? GROUP BY oids.id) AS oids ON oids.id=r.id WHERE rc.type=? ORDER BY res.result_id, res.ddate;");
// st = connection.prepareStatement("SELECT distinct res.result_id, r.title, r.publisher, r.source, rc.type, pids.pid, d.name, res.`date`, oids.oid, res.downloads, res.views "
// + "FROM (SELECT us.repository_id, us.result_id, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? "
// + "GROUP BY us.repository_id, us.result_id, us.`date`) AS res "
// + "JOIN " + statsDB + ".result r ON res.result_id=r.id JOIN " + statsDB + ".datasource d ON d.id=res.repository_id "
// + "JOIN " + statsDB + ".result_classifications rc ON rc.id=r.id "
// + "LEFT JOIN (SELECT pids.id, group_concat(pids.pid, '#!#') AS pid "
// + "FROM " + statsDB + ".result_pids pids, " + statsDB + ".result_datasources rd "
// + "WHERE rd.id=pids.id AND type='Digital Object Identifier' "
// + "AND rd.datasource=? GROUP BY pids.id) AS pids ON pids.id=r.id "
// + "LEFT JOIN (SELECT oids.id, group_concat(oids.oid, '#!#') AS oid "
// + "FROM " + statsDB + ".result_oids oids, " + statsDB + ".result_datasources rd "
// + "WHERE rd.id=oids.id AND rd.datasource=? GROUP BY oids.id) AS oids ON oids.id=r.id "
// + "WHERE rc.type=? ORDER BY res.result_id, res.`date`;");
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, repositoryIdentifier);
// //st.setString(4, beginDateStr);
// //st.setString(5, endDateStr);
// //st.setString(6, repositoryIdentifier);
// st.setString(4, repositoryIdentifier);
// st.setString(5, repositoryIdentifier);
// st.setString(6, itemDataType);
// }
// //log.error("IR STATEMENT: " + st);
//
// /*
// String redis_key = MD5(st.toString());
//
// if (jedis.hasKey(redis_key, "result")) {
// reportItems.addAll(reportItemsFromJson((String) jedis.get(redis_key, "result")));
// st.close();
// connection.close();
// return;
// }
// */
// rs = st.executeQuery();
// String result = "";
// String lastDate = "";
// COUNTER_Platform_Usage reportItem = null;
//
// int ft_total = 0;
// int abstr = 0;
// if (granularity.equalsIgnoreCase("totals")) {
// while (rs.next()) {
// if (!rs.getString(1).equals(result)) {
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), Integer.toString(ft_total), Integer.toString(abstr)));
// reportItems.add(reportItem);
// }
// result = rs.getString(1);
// reportItem = new COUNTER_Platform_Usage(rs.getString(3), rs.getString(7), rs.getString(5), rs.getString(2), "");
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OpenAIRE", rs.getString(1)));
// reportItem.addIdentifier(new COUNTER_Item_Identifier("URLs", rs.getString(4)));
// if (rs.getString(9) != null && !rs.getString(9).equals("")) {
// if (rs.getString(9).contains("#!#")) {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OAI", rs.getString(9).substring(0, rs.getString(9).indexOf("#!#"))));
// } else {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OAI", rs.getString(9)));
// }
// }
// if (rs.getString(6) != null && !rs.getString(6).equals("")) {
// if (rs.getString(6).contains("#!#")) {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("DOI", rs.getString(6).substring(0, rs.getString(6).indexOf("#!#"))));
// } else {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("DOI", rs.getString(6)));
// }
// }
// ft_total = 0;
// abstr = 0;
// }
// ft_total += rs.getInt(10);
// abstr += rs.getInt(11);
// }
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate), Integer.toString(ft_total), Integer.toString(abstr)));
// reportItems.add(reportItem);
// }
// } else if (granularity.equalsIgnoreCase("monthly")) {
// Calendar endCal = Calendar.getInstance();
// endCal.setTime(postgresFormat.parse(endDateStr));
// endCal.add(Calendar.MONTH, 1);
// Date endDateForZeros = endCal.getTime();
// while (rs.next()) {
// if (!rs.getString(1).equals(result)) {
// if (reportItem != null) {
// fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
// reportItems.add(reportItem);
// }
// result = rs.getString(1);
// lastDate = beginDateStr;
// reportItem = new COUNTER_Platform_Usage(rs.getString(3), rs.getString(7), rs.getString(5), rs.getString(2), "");
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OpenAIRE", rs.getString(1)));
// reportItem.addIdentifier(new COUNTER_Item_Identifier("URLs", rs.getString(4)));
// if (rs.getString(9) != null && !rs.getString(9).equals("")) {
// if (rs.getString(9).contains("#!#")) {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OAI", rs.getString(9).substring(0, rs.getString(9).indexOf("#!#"))));
// } else {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("OAI", rs.getString(9)));
// }
// }
// if (rs.getString(6) != null && !rs.getString(6).equals("")) {
// if (rs.getString(6).contains("#!#")) {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("DOI", rs.getString(6).substring(0, rs.getString(6).indexOf("#!#"))));
// } else {
// reportItem.addIdentifier(new COUNTER_Item_Identifier("DOI", rs.getString(6)));
// }
// }
// }
// fillWithZeros(postgresFormat.parse(lastDate), postgresFormat.parse(rs.getString(8)), reportItem);
// Calendar endC = Calendar.getInstance();
// endC.setTime(postgresFormat.parse(rs.getString(8)));
// endC.set(Calendar.DATE, endC.getActualMaximum(Calendar.DATE));
// if (reportItem != null) {
// reportItem.addPerformance(new COUNTER_Item_Performance(report_dateFormat.format(postgresFormat.parse(rs.getString(8))), report_dateFormat.format(endC.getTime()), rs.getString(10), rs.getString(11)));
// }
// endC.setTime(postgresFormat.parse(rs.getString(8)));
// endC.add(Calendar.MONTH, 1);
// lastDate = postgresFormat.format(endC.getTime());
// }
// if (reportItem != null) {
// fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
// reportItems.add(reportItem);
// }
// }
//
// /*
// jedis.put(redis_key, "persistent", "false");
// jedis.put(redis_key, "query", st.toString());
// jedis.put(redis_key, "result", toJson(reportItems));
// jedis.put(redis_key, "fetchMode", "3");
// */
// } catch (Exception e) {
// log.error("Batch Item Report failed: ", e);
// } finally {
// DbUtils.closeQuietly(rs);
// DbUtils.closeQuietly(st);
// DbUtils.closeQuietly(connection);
// }
// }
2021-02-25 16:33:44 +01:00
2021-01-25 09:44:27 +01:00
public void executeBatchItemsPR ( List < COUNTER_Platform_Usage > reportItems ,
2021-01-05 13:22:12 +01:00
String repositoryIdentifier , Date beginDate ,
2021-01-25 09:44:27 +01:00
Date endDate , String metricType , String dataType , String granularity ) {
2021-01-05 13:22:12 +01:00
SimpleDateFormat report_dateFormat = new SimpleDateFormat ( " yyyy-MM-dd " ) ;
SimpleDateFormat postgresFormat = new SimpleDateFormat ( " yyyy/MM " ) ;
String beginDateStr = postgresFormat . format ( beginDate ) ;
String endDateStr = postgresFormat . format ( endDate ) ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
2021-02-22 18:39:27 +01:00
String platform = getInstitutionName ( repositoryIdentifier ) ;
log . info ( " Platform " + platform ) ;
2021-03-16 06:56:18 +01:00
log . info ( " Data Type " + dataType ) ;
2021-01-05 13:22:12 +01:00
try {
connection = usageStatsDB . getConnection ( ) ;
2021-01-06 11:34:30 +01:00
if ( granularity . equalsIgnoreCase ( " totals " ) ) {
2021-03-14 15:24:14 +01:00
if ( dataType . equals ( " " ) | | dataType . equals ( " All " ) ) {
st = connection . prepareStatement ( " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, "
2021-01-25 09:44:27 +01:00
+ statsDB + " .result_classifications rc WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id GROUP BY rc.type order by rc.type ASC; " ) ;
2021-01-05 13:22:12 +01:00
2021-01-25 09:44:27 +01:00
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
2021-01-05 13:22:12 +01:00
2021-01-25 09:44:27 +01:00
rs = st . executeQuery ( ) ;
COUNTER_Platform_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
2021-01-25 09:44:27 +01:00
}
2021-03-14 15:24:14 +01:00
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
while ( rs . next ( ) ) {
reportItem = new COUNTER_Platform_Usage ( platform , rs . getString ( 1 ) , " Regular " ) ;
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 5 ) : null ) ) ;
reportItems . add ( reportItem ) ;
}
2021-01-25 09:44:27 +01:00
} else {
2021-03-14 15:24:14 +01:00
st = connection . prepareStatement ( " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, "
2021-01-25 09:44:27 +01:00
+ statsDB + " .result_classifications rc WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.type=? AND rc.id=us.result_id GROUP BY rc.type order by rc.type ASC; " ) ;
2021-01-05 13:22:12 +01:00
2021-01-25 09:44:27 +01:00
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
st . setString ( 4 , dataType ) ;
rs = st . executeQuery ( ) ;
COUNTER_Platform_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
2021-01-25 09:44:27 +01:00
2021-03-14 15:24:14 +01:00
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
while ( rs . next ( ) ) {
reportItem = new COUNTER_Platform_Usage ( platform , rs . getString ( 1 ) , " Regular " ) ;
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 5 ) : null ) ) ;
reportItems . add ( reportItem ) ;
2021-01-25 09:44:27 +01:00
}
2021-03-14 15:24:14 +01:00
2021-01-05 13:22:12 +01:00
}
} else if ( granularity . equalsIgnoreCase ( " monthly " ) ) {
2021-03-14 15:24:14 +01:00
if ( dataType . equals ( " " ) | | dataType . equals ( " All " ) ) {
st = connection . prepareStatement ( " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, "
2021-01-25 09:44:27 +01:00
+ statsDB + " .result_classifications rc WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id GROUP BY rc.type, us.`date` order by rc.type, us.`date` ASC; " ) ;
2021-01-06 11:34:30 +01:00
2021-01-25 09:44:27 +01:00
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
2021-01-06 11:34:30 +01:00
2021-01-25 09:44:27 +01:00
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
COUNTER_Platform_Usage reportItem = null ;
2021-01-06 11:34:30 +01:00
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
2021-01-25 09:44:27 +01:00
int ft_total = 0 ;
int abstr = 0 ;
2021-01-06 11:34:30 +01:00
2021-01-25 09:44:27 +01:00
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
2021-01-06 11:34:30 +01:00
2021-01-25 09:44:27 +01:00
lastDate = beginDateStr ;
String datatype = " " ;
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
//iterate on data types
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
if ( reportItem ! = null ) {
reportItems . add ( reportItem ) ;
}
2021-02-22 18:39:27 +01:00
reportItem = new COUNTER_Platform_Usage ( platform , rs . getString ( 1 ) , " Regular " ) ;
2021-01-25 09:44:27 +01:00
datatype = rs . getString ( 1 ) ;
}
2021-01-06 11:34:30 +01:00
if ( reportItem ! = null ) {
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 5 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 6 ) : null ) ) ;
2021-01-25 09:44:27 +01:00
2021-01-06 11:34:30 +01:00
}
2021-01-25 09:44:27 +01:00
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
2021-01-06 11:34:30 +01:00
}
2021-01-25 09:44:27 +01:00
//add last report item
reportItems . add ( reportItem ) ;
} else {
2021-03-14 15:24:14 +01:00
st = connection . prepareStatement ( " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, "
2021-01-25 09:44:27 +01:00
+ statsDB + " .result_classifications rc WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.type=? AND rc.id=us.result_id GROUP BY rc.type, us.`date` order by rc.type, us.`date` ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
st . setString ( 4 , dataType ) ;
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
COUNTER_Platform_Usage reportItem = null ;
2021-01-13 12:01:47 +01:00
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
2021-01-25 09:44:27 +01:00
int ft_total = 0 ;
int abstr = 0 ;
2021-01-13 12:01:47 +01:00
2021-01-25 09:44:27 +01:00
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
2021-01-13 12:01:47 +01:00
2021-01-25 09:44:27 +01:00
lastDate = beginDateStr ;
String datatype = " " ;
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
//iterate on data types
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
if ( reportItem ! = null ) {
reportItems . add ( reportItem ) ;
}
2021-02-22 18:39:27 +01:00
reportItem = new COUNTER_Platform_Usage ( platform , rs . getString ( 1 ) , " Regular " ) ;
2021-01-25 09:44:27 +01:00
datatype = rs . getString ( 1 ) ;
2021-01-13 12:01:47 +01:00
}
2021-01-25 09:44:27 +01:00
if ( reportItem ! = null ) {
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 5 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 6 ) : null ) ) ;
2021-01-25 09:44:27 +01:00
}
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
2021-01-05 13:22:12 +01:00
}
2021-01-25 09:44:27 +01:00
//add last report item
reportItems . add ( reportItem ) ;
2021-01-05 13:22:12 +01:00
}
2021-01-25 09:44:27 +01:00
2021-01-05 13:22:12 +01:00
}
2020-12-30 14:56:37 +01:00
/ *
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " query " , st . toString ( ) ) ;
jedis . put ( redis_key , " result " , toJson ( reportItems ) ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
* /
} catch ( Exception e ) {
log . error ( " Batch Item Report failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
}
2021-01-25 09:44:27 +01:00
public void executeBatchItemsPR_P1 ( List < COUNTER_Platform_Usage > reportItems ,
2021-01-06 15:27:29 +01:00
String repositoryIdentifier , Date beginDate ,
Date endDate ) {
SimpleDateFormat report_dateFormat = new SimpleDateFormat ( " yyyy-MM-dd " ) ;
SimpleDateFormat postgresFormat = new SimpleDateFormat ( " yyyy/MM " ) ;
String beginDateStr = postgresFormat . format ( beginDate ) ;
String endDateStr = postgresFormat . format ( endDate ) ;
2021-02-22 18:39:27 +01:00
String platform = getInstitutionName ( repositoryIdentifier ) ;
2021-01-06 15:27:29 +01:00
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
2021-03-14 15:24:14 +01:00
st = connection . prepareStatement ( " SELECT us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us "
2021-01-06 15:27:29 +01:00
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? GROUP BY us.`date` order by us.`date` ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
2021-01-25 09:44:27 +01:00
COUNTER_Platform_Usage reportItem = null ;
2021-01-06 15:27:29 +01:00
int ft_total = 0 ;
int abstr = 0 ;
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
lastDate = beginDateStr ;
String datatype = " " ;
2021-02-22 18:39:27 +01:00
reportItem = new COUNTER_Platform_Usage ( platform , null , " Regular " ) ;
2021-01-06 15:27:29 +01:00
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 1 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
//iterate on data types
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 1 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
rs . getString ( 2 ) , rs . getString ( 3 ) , rs . getString ( 4 ) , rs . getString ( 5 ) ) ) ;
2021-01-06 15:27:29 +01:00
endC . setTime ( postgresFormat . parse ( rs . getString ( 1 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
}
2021-01-13 12:01:47 +01:00
if ( reportItem . getItemPerformances ( ) . size ( ) > 0 ) {
2021-01-06 15:27:29 +01:00
reportItems . add ( reportItem ) ;
2021-01-13 12:01:47 +01:00
}
2021-01-06 15:27:29 +01:00
/ *
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " query " , st . toString ( ) ) ;
jedis . put ( redis_key , " result " , toJson ( reportItems ) ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
* /
} catch ( Exception e ) {
log . error ( " Batch Item Report failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
}
2021-02-22 18:39:27 +01:00
public void executeBatchItemsIR ( List < COUNTER_Item_Usage > reportItems ,
String repositoryIdentifier , String itemIdentifier , Date beginDate ,
2021-02-25 09:10:47 +01:00
Date endDate , List < String > metricType , String dataType , String granularity ) throws Exception {
2021-02-22 18:39:27 +01:00
SimpleDateFormat report_dateFormat = new SimpleDateFormat ( " yyyy-MM-dd " ) ;
SimpleDateFormat postgresFormat = new SimpleDateFormat ( " yyyy/MM " ) ;
String beginDateStr = postgresFormat . format ( beginDate ) ;
String endDateStr = postgresFormat . format ( endDate ) ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
String platform = getInstitutionName ( repositoryIdentifier ) ;
if ( itemIdentifier . equals ( " " ) ) {
try {
connection = usageStatsDB . getConnection ( ) ;
if ( granularity . equalsIgnoreCase ( " totals " ) ) {
if ( dataType . equals ( " " ) ) {
2021-02-25 09:10:47 +01:00
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
2021-03-14 15:24:14 +01:00
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
2021-02-25 09:10:47 +01:00
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id "
+ " GROUP BY rc.type, rs.title, us.result_id, rs.title, rs.year order by rc.type ASC; " ) ;
2021-02-22 18:39:27 +01:00
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
rs = st . executeQuery ( ) ;
COUNTER_Item_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
while ( rs . next ( ) ) {
reportItem = new COUNTER_Item_Usage ( rs . getString ( 5 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 6 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 7 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-02-25 09:10:47 +01:00
}
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 8 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 9 ) : null ) ) ;
reportItems . add ( reportItem ) ;
2021-02-22 18:39:27 +01:00
}
2021-03-14 15:24:14 +01:00
2021-02-22 18:39:27 +01:00
} else {
2021-02-25 09:10:47 +01:00
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
2021-03-14 15:24:14 +01:00
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
2021-02-25 09:10:47 +01:00
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.type=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year order by rc.type ASC; " ) ;
2021-02-22 18:39:27 +01:00
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
st . setString ( 4 , dataType ) ;
rs = st . executeQuery ( ) ;
COUNTER_Item_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
while ( rs . next ( ) ) {
reportItem = new COUNTER_Item_Usage ( rs . getString ( 5 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 6 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 7 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-02-25 09:10:47 +01:00
}
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 8 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 9 ) : null ) ) ;
reportItems . add ( reportItem ) ;
2021-02-22 18:39:27 +01:00
}
}
2021-02-25 09:10:47 +01:00
} else if ( granularity . equalsIgnoreCase ( " monthly " ) ) {
2021-02-22 18:39:27 +01:00
2021-02-25 09:10:47 +01:00
if ( dataType . equals ( " " ) ) {
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
2021-03-14 15:24:14 +01:00
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
2021-02-25 09:10:47 +01:00
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id "
+ " AND tpd.id=us.result_id "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
COUNTER_Item_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
2021-02-25 09:10:47 +01:00
int ft_total = 0 ;
int abstr = 0 ;
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
lastDate = beginDateStr ;
String datatype = " " ;
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
2021-02-22 18:39:27 +01:00
// //iterate on data types
2021-02-25 09:10:47 +01:00
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
datatype = rs . getString ( 1 ) ;
}
2021-03-14 15:24:14 +01:00
reportItem = new COUNTER_Item_Usage ( rs . getString ( 6 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 7 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 8 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-02-25 09:10:47 +01:00
}
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 9 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 10 ) : null ) ) ;
reportItems . add ( reportItem ) ;
2021-02-25 09:10:47 +01:00
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
}
} else {
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
2021-03-14 15:24:14 +01:00
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
2021-02-25 09:10:47 +01:00
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.type=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
st . setString ( 4 , dataType ) ;
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
COUNTER_Item_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
2021-02-25 09:10:47 +01:00
int ft_total = 0 ;
int abstr = 0 ;
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
lastDate = beginDateStr ;
String datatype = " " ;
List < COUNTER_Item_Identifiers > identifiers = null ;
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
2021-02-22 18:39:27 +01:00
// //iterate on data types
2021-02-25 09:10:47 +01:00
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
2021-02-22 18:39:27 +01:00
// if (reportItem != null) {
// reportItems.add(reportItem);
// }
// reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", "");
2021-02-25 09:10:47 +01:00
datatype = rs . getString ( 1 ) ;
}
2021-03-14 15:24:14 +01:00
reportItem = new COUNTER_Item_Usage ( rs . getString ( 6 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 7 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 8 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-02-25 09:10:47 +01:00
}
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 9 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 20 ) : null ) ) ;
reportItems . add ( reportItem ) ;
2021-02-25 09:10:47 +01:00
// }
//
// //}
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
}
}
}
2021-02-22 18:39:27 +01:00
/ *
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " query " , st . toString ( ) ) ;
jedis . put ( redis_key , " result " , toJson ( reportItems ) ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
* /
} catch ( Exception e ) {
log . error ( " Batch Item Report failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
2021-02-25 09:10:47 +01:00
} else {
try {
connection = usageStatsDB . getConnection ( ) ;
if ( granularity . equalsIgnoreCase ( " totals " ) ) {
if ( dataType . equals ( " " ) ) {
2021-02-25 16:33:44 +01:00
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
2021-03-14 15:24:14 +01:00
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
2021-02-25 16:33:44 +01:00
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id "
2021-02-25 09:10:47 +01:00
+ " GROUP BY rc.type, rs.title, us.result_id, rs.title, rs.year order by rc.type ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
st . setString ( 4 , itemIdentifier ) ;
rs = st . executeQuery ( ) ;
COUNTER_Item_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
while ( rs . next ( ) ) {
reportItem = new COUNTER_Item_Usage ( rs . getString ( 5 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 6 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 7 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-02-25 09:10:47 +01:00
}
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 8 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 9 ) : null ) ) ;
reportItems . add ( reportItem ) ;
2021-02-25 09:10:47 +01:00
}
} else {
2021-02-25 16:33:44 +01:00
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
2021-03-14 15:24:14 +01:00
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
2021-02-25 16:33:44 +01:00
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd "
2021-02-25 09:10:47 +01:00
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.type=? AND rc.id=us.result_id "
2021-02-25 16:33:44 +01:00
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year order by rc.type ASC; " ) ;
2021-02-25 09:10:47 +01:00
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
st . setString ( 4 , itemIdentifier ) ;
st . setString ( 5 , dataType ) ;
rs = st . executeQuery ( ) ;
COUNTER_Item_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Items_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Items_Ivestigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
while ( rs . next ( ) ) {
reportItem = new COUNTER_Item_Usage ( rs . getString ( 5 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 6 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 7 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-02-25 09:10:47 +01:00
}
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 8 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 9 ) : null ) ) ;
reportItems . add ( reportItem ) ;
2021-02-25 09:10:47 +01:00
}
}
} else if ( granularity . equalsIgnoreCase ( " monthly " ) ) {
if ( dataType . equals ( " " ) ) {
2021-02-25 16:33:44 +01:00
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
2021-03-14 15:24:14 +01:00
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
2021-02-25 16:33:44 +01:00
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id "
+ " AND tpd.id=us.result_id "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC; " ) ;
2021-02-25 09:10:47 +01:00
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
st . setString ( 4 , itemIdentifier ) ;
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
COUNTER_Item_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
2021-02-25 09:10:47 +01:00
int ft_total = 0 ;
int abstr = 0 ;
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
lastDate = beginDateStr ;
String datatype = " " ;
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
2021-02-22 18:39:27 +01:00
// //iterate on data types
2021-02-25 09:10:47 +01:00
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
datatype = rs . getString ( 1 ) ;
}
2021-03-14 15:24:14 +01:00
reportItem = new COUNTER_Item_Usage ( rs . getString ( 6 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 7 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 8 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-02-25 09:10:47 +01:00
}
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 9 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 10 ) : null ) ) ;
reportItems . add ( reportItem ) ;
2021-02-25 09:10:47 +01:00
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
}
} else {
2021-02-25 16:33:44 +01:00
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
2021-03-14 15:24:14 +01:00
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
2021-02-25 16:33:44 +01:00
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.type=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year order by rc.type, us.`date` ASC; " ) ;
2021-02-25 09:10:47 +01:00
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
st . setString ( 4 , itemIdentifier ) ;
st . setString ( 5 , dataType ) ;
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
COUNTER_Item_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
2021-02-25 09:10:47 +01:00
int ft_total = 0 ;
int abstr = 0 ;
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
lastDate = beginDateStr ;
String datatype = " " ;
List < COUNTER_Item_Identifiers > identifiers = null ;
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
2021-02-22 18:39:27 +01:00
// //iterate on data types
2021-02-25 09:10:47 +01:00
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
2021-02-22 18:39:27 +01:00
// if (reportItem != null) {
// reportItems.add(reportItem);
// }
// reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", "");
2021-02-25 09:10:47 +01:00
datatype = rs . getString ( 1 ) ;
}
2021-03-14 15:24:14 +01:00
reportItem = new COUNTER_Item_Usage ( rs . getString ( 6 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 7 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 8 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-02-25 09:10:47 +01:00
}
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 9 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 10 ) : null ) ) ;
reportItems . add ( reportItem ) ;
2021-02-25 09:10:47 +01:00
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
}
}
}
/ *
2021-02-22 18:39:27 +01:00
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " query " , st . toString ( ) ) ;
jedis . put ( redis_key , " result " , toJson ( reportItems ) ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
2021-02-25 09:10:47 +01:00
* /
} catch ( Exception e ) {
log . error ( " Batch Item Report failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
2021-02-22 18:39:27 +01:00
}
}
2021-03-14 15:24:14 +01:00
public void executeItemsAllRepoIR ( List < COUNTER_Item_Usage > reportItems ,
String itemIdentifier , Date beginDate ,
Date endDate , List < String > metricType , String dataType , String granularity ) throws Exception {
2020-12-30 14:56:37 +01:00
SimpleDateFormat report_dateFormat = new SimpleDateFormat ( " yyyy-MM-dd " ) ;
2021-03-14 15:24:14 +01:00
SimpleDateFormat postgresFormat = new SimpleDateFormat ( " yyyy/MM " ) ;
String beginDateStr = postgresFormat . format ( beginDate ) ;
String endDateStr = postgresFormat . format ( endDate ) ;
2020-12-30 14:56:37 +01:00
2021-03-14 15:24:14 +01:00
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
2020-12-30 14:56:37 +01:00
2021-03-14 15:24:14 +01:00
try {
connection = usageStatsDB . getConnection ( ) ;
if ( granularity . equalsIgnoreCase ( " totals " ) ) {
if ( dataType . equals ( " " ) ) {
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd, " + statsDB + " .datasource ds "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id "
+ " GROUP BY rc.type, rs.title, us.result_id, rs.title, rs.year, ds.name order by rc.type ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , itemIdentifier ) ;
rs = st . executeQuery ( ) ;
COUNTER_Item_Usage reportItem = null ;
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
while ( rs . next ( ) ) {
reportItem = new COUNTER_Item_Usage ( rs . getString ( 5 ) , " OpenAIRE " , rs . getString ( 8 ) , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 6 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 7 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
}
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 9 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 10 ) : null ) ) ;
reportItems . add ( reportItem ) ;
}
} else {
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd, " + statsDB + " .datasource ds "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.type=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year, ds.name order by rc.type ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , itemIdentifier ) ;
st . setString ( 4 , dataType ) ;
rs = st . executeQuery ( ) ;
COUNTER_Item_Usage reportItem = null ;
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
while ( rs . next ( ) ) {
String platform = getInstitutionName ( rs . getString ( 8 ) ) ;
reportItem = new COUNTER_Item_Usage ( rs . getString ( 5 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 6 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 7 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
}
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 9 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 10 ) : null ) ) ;
reportItems . add ( reportItem ) ;
}
}
} else if ( granularity . equalsIgnoreCase ( " monthly " ) ) {
if ( dataType . equals ( " " ) ) {
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd, " + statsDB + " .datasource ds "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id "
+ " AND tpd.id=us.result_id "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year, ds.name order by rc.type, us.`date` ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , itemIdentifier ) ;
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
COUNTER_Item_Usage reportItem = null ;
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
int ft_total = 0 ;
int abstr = 0 ;
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
lastDate = beginDateStr ;
String datatype = " " ;
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
// //iterate on data types
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
datatype = rs . getString ( 1 ) ;
}
reportItem = new COUNTER_Item_Usage ( rs . getString ( 6 ) , " OpenAIRE " , rs . getString ( 9 ) , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 7 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 8 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
}
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 10 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 11 ) : null ) ) ;
reportItems . add ( reportItem ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
}
} else {
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd, " + statsDB + " .datasource ds "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.type=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year, ds.name order by rc.type, us.`date` ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , itemIdentifier ) ;
st . setString ( 4 , dataType ) ;
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
COUNTER_Item_Usage reportItem = null ;
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Item_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Item_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Item_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
int ft_total = 0 ;
int abstr = 0 ;
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
lastDate = beginDateStr ;
String datatype = " " ;
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
// //iterate on data types
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
// if (reportItem != null) {
// reportItems.add(reportItem);
// }
// reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", "");
datatype = rs . getString ( 1 ) ;
}
reportItem = new COUNTER_Item_Usage ( rs . getString ( 6 ) , " OpenAIRE " , rs . getString ( 9 ) , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 7 ) ) , null , " Regular " ) ;
String [ ] identifiersAll = rs . getString ( 8 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Item_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
}
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 10 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 11 ) : null ) ) ;
reportItems . add ( reportItem ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
}
}
}
/ *
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " query " , st . toString ( ) ) ;
jedis . put ( redis_key , " result " , toJson ( reportItems ) ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
* /
} catch ( Exception e ) {
log . error ( " Batch Item Report failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
}
public void executeBatchItemsDSR ( List < COUNTER_Dataset_Usage > reportItems ,
String repositoryIdentifier , String itemIdentifier , Date beginDate ,
2021-03-16 06:56:18 +01:00
Date endDate , List < String > metricType , String granularity ) throws Exception {
2021-03-14 15:24:14 +01:00
SimpleDateFormat report_dateFormat = new SimpleDateFormat ( " yyyy-MM-dd " ) ;
SimpleDateFormat postgresFormat = new SimpleDateFormat ( " yyyy/MM " ) ;
String beginDateStr = postgresFormat . format ( beginDate ) ;
String endDateStr = postgresFormat . format ( endDate ) ;
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
String platform = getInstitutionName ( repositoryIdentifier ) ;
if ( itemIdentifier . equals ( " " ) ) {
try {
connection = usageStatsDB . getConnection ( ) ;
if ( granularity . equalsIgnoreCase ( " totals " ) ) {
2021-03-16 06:56:18 +01:00
// if (dataType.equals("")) {
// st = connection.prepareStatement("WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + ".result_pids) "
// + "SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
// + "us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
// + "count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews, "
// + "dp.access_method as access_method "
// + "FROM " + usagestatsImpalaDB + ".usage_stats us, " + statsDB + ".result_classifications rc, " + statsDB + ".result rs, "
// + statsDB + ".result_oids ro, tpd, datasetsusagestats_20210312a.datasetsperformance dp "
// + "WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id "
// + "AND dp.ds_type=ro.oid "
// + "GROUP BY rc.type, rs.title, us.result_id, rs.title, rs.year, dp.access_method order by rc.type ASC;");
//
// st.setString(1, beginDateStr);
// st.setString(2, endDateStr);
// st.setString(3, repositoryIdentifier);
//
// rs = st.executeQuery();
// COUNTER_Dataset_Usage reportItem = null;
//
// boolean[] metricTypeValue = {false, false, false, false};
// if (metricType.contains("Total_Dataset_Requests")) {
// metricTypeValue[0] = true;
// }
// if (metricType.contains("Total_Dataset_Investigations")) {
// metricTypeValue[1] = true;
// }
// if (metricType.contains("Unique_Dataset_Requests")) {
// metricTypeValue[2] = true;
// }
// if (metricType.contains("Unique_Dataset_Investigations")) {
// metricTypeValue[3] = true;
// }
//
// while (rs.next()) {
// reportItem = new COUNTER_Dataset_Usage(rs.getString(5), "OpenAIRE", platform, rs.getString(1), Integer.toString(rs.getInt(6)), rs.getString(10));
// String[] identifiersAll = rs.getString(7).split("#!#");
// for (int i = 0; i < identifiersAll.length; i++) {
// String[] typeIdentifierArray = identifiersAll[i].split("#-#");
// reportItem.addIdentifier(new COUNTER_Dataset_Identifiers(typeIdentifierArray[0], typeIdentifierArray[1]));
// }
//
// reportItem.addPerformance(new COUNTER_Dataset_Performance(report_dateFormat.format(beginDate), report_dateFormat.format(endDate),
// metricTypeValue[0] == true ? rs.getString(2) : null, metricTypeValue[1] == true ? rs.getString(3) : null, metricTypeValue[2] == true ? rs.getString(8) : null, metricTypeValue[3] == true ? rs.getString(9) : null));
// reportItems.add(reportItem);
// }
//
// } //else {
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews, "
+ " dp.access_method as access_method "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd, datasetsusagestats_20210312a.datasetsperformance dp "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id AND rc.type='Dataset' "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year, dp.access_method ORDER by rc.type ASC; " ) ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
rs = st . executeQuery ( ) ;
COUNTER_Dataset_Usage reportItem = null ;
boolean [ ] metricTypeValue = { false , false , false , false } ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
if ( metricType . contains ( " Total_Dataset_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Dataset_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
while ( rs . next ( ) ) {
reportItem = new COUNTER_Dataset_Usage ( rs . getString ( 5 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 6 ) ) , rs . getString ( 10 ) ) ;
String [ ] identifiersAll = rs . getString ( 7 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Dataset_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-03-14 15:24:14 +01:00
}
2021-03-16 06:56:18 +01:00
reportItem . addPerformance ( new COUNTER_Dataset_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 8 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 9 ) : null ) ) ;
reportItems . add ( reportItem ) ;
}
//}
} else if ( granularity . equalsIgnoreCase ( " monthly " ) ) {
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews, "
+ " dp.access_method as access_method "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd, datasetsusagestats_20210312a.datasetsperformance dp "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id AND rc.type='Dataset' "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year, dp.access_method ORDER by rc.type, us.`date` ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
rs = st . executeQuery ( ) ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
String result = " " ;
String lastDate = " " ;
COUNTER_Dataset_Usage reportItem = null ;
boolean [ ] metricTypeValue = { false , false , false , false } ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
if ( metricType . contains ( " Total_Dataset_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Dataset_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
int ft_total = 0 ;
int abstr = 0 ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
lastDate = beginDateStr ;
String datatype = " " ;
List < COUNTER_Item_Identifiers > identifiers = null ;
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
// //iterate on data types
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
// if (reportItem != null) {
// reportItems.add(reportItem);
// }
// reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", "");
datatype = rs . getString ( 1 ) ;
2021-03-14 15:24:14 +01:00
}
2021-03-16 06:56:18 +01:00
reportItem = new COUNTER_Dataset_Usage ( rs . getString ( 6 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 7 ) ) , rs . getString ( 11 ) ) ;
String [ ] identifiersAll = rs . getString ( 8 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Dataset_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-03-14 15:24:14 +01:00
}
2021-03-16 06:56:18 +01:00
reportItem . addPerformance ( new COUNTER_Dataset_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 9 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 10 ) : null ) ) ;
reportItems . add ( reportItem ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
2021-03-14 15:24:14 +01:00
}
2021-03-16 06:56:18 +01:00
}
/ *
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " query " , st . toString ( ) ) ;
jedis . put ( redis_key , " result " , toJson ( reportItems ) ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
* /
} catch ( Exception e ) {
log . error ( " Batch Item Report failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
} else {
try {
connection = usageStatsDB . getConnection ( ) ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
if ( granularity . equalsIgnoreCase ( " totals " ) ) {
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews, "
+ " dp.access_method as access_method "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd, datasetsusagestats_20210312a.datasetsperformance dp "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id AND rc.type='Dataset' "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year, dp.access_method ORDER by rc.type ASC; " ) ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
st . setString ( 4 , itemIdentifier ) ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
rs = st . executeQuery ( ) ;
COUNTER_Dataset_Usage reportItem = null ;
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Dataset_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Dataset_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
while ( rs . next ( ) ) {
reportItem = new COUNTER_Dataset_Usage ( rs . getString ( 5 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 6 ) ) , rs . getString ( 10 ) ) ;
String [ ] identifiersAll = rs . getString ( 7 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Dataset_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-03-14 15:24:14 +01:00
}
2021-03-16 06:56:18 +01:00
reportItem . addPerformance ( new COUNTER_Dataset_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 8 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 9 ) : null ) ) ;
reportItems . add ( reportItem ) ;
}
} else if ( granularity . equalsIgnoreCase ( " monthly " ) ) {
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews, "
+ " dp.access_method as access_method "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd, datasetsusagestats_20210312a.datasetsperformance dp "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=? AND us.result_id=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id AND rc.type='Dataset' "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year, dp.access_method ORDER by rc.type, us.`date` ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , repositoryIdentifier ) ;
st . setString ( 4 , itemIdentifier ) ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
COUNTER_Dataset_Usage reportItem = null ;
2021-03-14 15:24:14 +01:00
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Dataset_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Dataset_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
int ft_total = 0 ;
int abstr = 0 ;
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
lastDate = beginDateStr ;
String datatype = " " ;
2021-03-16 06:56:18 +01:00
2021-03-14 15:24:14 +01:00
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
// //iterate on data types
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
datatype = rs . getString ( 1 ) ;
}
2021-03-16 06:56:18 +01:00
reportItem = new COUNTER_Dataset_Usage ( rs . getString ( 6 ) , " OpenAIRE " , platform , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 7 ) ) , rs . getString ( 11 ) ) ;
2021-03-14 15:24:14 +01:00
String [ ] identifiersAll = rs . getString ( 8 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Dataset_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
}
reportItem . addPerformance ( new COUNTER_Dataset_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
2021-03-16 06:56:18 +01:00
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 9 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 10 ) : null ) ) ;
2021-03-14 15:24:14 +01:00
reportItems . add ( reportItem ) ;
2021-03-16 06:56:18 +01:00
2021-03-14 15:24:14 +01:00
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
}
2021-03-16 06:56:18 +01:00
}
/ *
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " query " , st . toString ( ) ) ;
jedis . put ( redis_key , " result " , toJson ( reportItems ) ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
* /
} catch ( Exception e ) {
log . error ( " Batch Item Report failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
}
}
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
public void executeItemsAllRepoDSR ( List < COUNTER_Dataset_Usage > reportItems ,
String itemIdentifier , Date beginDate ,
Date endDate , List < String > metricType , String granularity ) throws Exception {
SimpleDateFormat report_dateFormat = new SimpleDateFormat ( " yyyy-MM-dd " ) ;
SimpleDateFormat postgresFormat = new SimpleDateFormat ( " yyyy/MM " ) ;
String beginDateStr = postgresFormat . format ( beginDate ) ;
String endDateStr = postgresFormat . format ( endDate ) ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
Connection connection = null ;
PreparedStatement st = null ;
ResultSet rs = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
if ( granularity . equalsIgnoreCase ( " totals " ) ) {
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews, "
+ " dp.access_method "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd, " + statsDB + " .datasource ds, datasetsusagestats_20210312a.datasetsperformance dp "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id AND rc.type='Dataset' "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, rs.year, ds.name,dp.access_method ORDER by rc.type ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , itemIdentifier ) ;
rs = st . executeQuery ( ) ;
COUNTER_Dataset_Usage reportItem = null ;
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Dataset_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Dataset_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
while ( rs . next ( ) ) {
String platform = getInstitutionName ( rs . getString ( 8 ) ) ;
reportItem = new COUNTER_Dataset_Usage ( rs . getString ( 5 ) , " OpenAIRE " , rs . getString ( 8 ) , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 6 ) ) , rs . getString ( 11 ) ) ;
String [ ] identifiersAll = rs . getString ( 7 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Dataset_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
2021-03-14 15:24:14 +01:00
}
2021-03-16 06:56:18 +01:00
reportItem . addPerformance ( new COUNTER_Dataset_Performance ( report_dateFormat . format ( beginDate ) , report_dateFormat . format ( endDate ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 2 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 9 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 10 ) : null ) ) ;
reportItems . add ( reportItem ) ;
}
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
//}
} else if ( granularity . equalsIgnoreCase ( " monthly " ) ) {
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
st = connection . prepareStatement ( " WITH tpd as (select id, concat(type,'#-#',pid) type_id from " + statsDB + " .result_pids) "
+ " SELECT rc.type, us.`date`, sum(us.downloads) as downloads, sum(us.views) as views, "
+ " us.result_id as resultid, rs.title as item, rs.year as yop, group_concat(distinct tpd.type_id,'#!#') as oid, ds.name, "
+ " count(case when us.downloads >0 then 1 else null end) as uniquedownloads, count(case when us.views >0 then 1 else null end) as uniqueviews, "
+ " dp.access_method "
+ " FROM " + usagestatsImpalaDB + " .usage_stats us, " + statsDB + " .result_classifications rc, " + statsDB + " .result rs, "
+ statsDB + " .result_oids ro, tpd, " + statsDB + " .datasource ds, datasetsusagestats_20210312a.datasetsperformance dp "
+ " WHERE us.`date`>=? AND us.`date`<=? AND us.repository_id=ds.id AND us.result_id=? AND rc.id=us.result_id "
+ " AND us.result_id=rs.id AND ro.id=us.result_id AND tpd.id=us.result_id AND rc.type='Dataset' "
+ " GROUP BY ro.id, rc.type, rs.title, us.result_id, us.`date`, rs.year, ds.name, dp.access_method ORDER by rc.type, us.`date` ASC; " ) ;
st . setString ( 1 , beginDateStr ) ;
st . setString ( 2 , endDateStr ) ;
st . setString ( 3 , itemIdentifier ) ;
rs = st . executeQuery ( ) ;
String result = " " ;
String lastDate = " " ;
COUNTER_Dataset_Usage reportItem = null ;
boolean [ ] metricTypeValue = { false , false , false , false } ;
if ( metricType . contains ( " Total_Dataset_Requests " ) ) {
metricTypeValue [ 0 ] = true ;
}
if ( metricType . contains ( " Total_Dataset_Investigations " ) ) {
metricTypeValue [ 1 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Requests " ) ) {
metricTypeValue [ 2 ] = true ;
}
if ( metricType . contains ( " Unique_Dataset_Investigations " ) ) {
metricTypeValue [ 3 ] = true ;
}
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
int ft_total = 0 ;
int abstr = 0 ;
Calendar endCal = Calendar . getInstance ( ) ;
endCal . setTime ( postgresFormat . parse ( endDateStr ) ) ;
endCal . add ( Calendar . MONTH , 1 ) ;
Date endDateForZeros = endCal . getTime ( ) ;
lastDate = beginDateStr ;
String datatype = " " ;
while ( rs . next ( ) ) {
Calendar endC = Calendar . getInstance ( ) ;
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . set ( Calendar . DATE , endC . getActualMaximum ( Calendar . DATE ) ) ;
2021-03-14 15:24:14 +01:00
// //iterate on data types
2021-03-16 06:56:18 +01:00
if ( ! datatype . equals ( rs . getString ( 1 ) ) ) {
2021-03-14 15:24:14 +01:00
// if (reportItem != null) {
// reportItems.add(reportItem);
// }
// reportItem = new COUNTER_Platform_Usage("", "OpenAIRE", rs.getString(1), "Regular", "");
2021-03-16 06:56:18 +01:00
datatype = rs . getString ( 1 ) ;
}
reportItem = new COUNTER_Dataset_Usage ( rs . getString ( 6 ) , " OpenAIRE " , rs . getString ( 9 ) , rs . getString ( 1 ) , Integer . toString ( rs . getInt ( 7 ) ) , rs . getString ( 12 ) ) ;
String [ ] identifiersAll = rs . getString ( 8 ) . split ( " #!# " ) ;
for ( int i = 0 ; i < identifiersAll . length ; i + + ) {
String [ ] typeIdentifierArray = identifiersAll [ i ] . split ( " #-# " ) ;
reportItem . addIdentifier ( new COUNTER_Dataset_Identifiers ( typeIdentifierArray [ 0 ] , typeIdentifierArray [ 1 ] ) ) ;
}
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
reportItem . addPerformance ( new COUNTER_Dataset_Performance ( report_dateFormat . format ( postgresFormat . parse ( rs . getString ( 2 ) ) ) , report_dateFormat . format ( endC . getTime ( ) ) ,
metricTypeValue [ 0 ] = = true ? rs . getString ( 3 ) : null , metricTypeValue [ 1 ] = = true ? rs . getString ( 4 ) : null , metricTypeValue [ 2 ] = = true ? rs . getString ( 10 ) : null , metricTypeValue [ 3 ] = = true ? rs . getString ( 11 ) : null ) ) ;
reportItems . add ( reportItem ) ;
2021-03-14 15:24:14 +01:00
2021-03-16 06:56:18 +01:00
endC . setTime ( postgresFormat . parse ( rs . getString ( 2 ) ) ) ;
endC . add ( Calendar . MONTH , 1 ) ;
lastDate = postgresFormat . format ( endC . getTime ( ) ) ;
//if (reportItem != null) {
//fillWithZeros(postgresFormat.parse(lastDate), endDateForZeros, reportItem);
2021-03-14 15:24:14 +01:00
}
}
/ *
jedis . put ( redis_key , " persistent " , " false " ) ;
jedis . put ( redis_key , " query " , st . toString ( ) ) ;
jedis . put ( redis_key , " result " , toJson ( reportItems ) ) ;
jedis . put ( redis_key , " fetchMode " , " 3 " ) ;
* /
} catch ( Exception e ) {
log . error ( " Batch Item Report failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
}
private void fillWithZeros ( Date from , Date to , COUNTER_Platform_Usage reportItem ) {
SimpleDateFormat report_dateFormat = new SimpleDateFormat ( " yyyy-MM-dd " ) ;
Calendar fromCalendar = Calendar . getInstance ( ) ;
fromCalendar . setTime ( from ) ;
Calendar toCalendar = Calendar . getInstance ( ) ;
toCalendar . setTime ( to ) ;
while ( from . before ( to ) ) {
Calendar temp_c = Calendar . getInstance ( ) ;
temp_c . setTime ( from ) ;
temp_c . set ( Calendar . DAY_OF_MONTH , temp_c . getActualMaximum ( Calendar . DAY_OF_MONTH ) ) ;
Date temp_endDate = temp_c . getTime ( ) ;
2020-12-30 14:56:37 +01:00
2021-03-14 15:24:14 +01:00
reportItem . addPerformance ( new COUNTER_Item_Performance ( report_dateFormat . format ( from ) , report_dateFormat . format ( temp_endDate ) , " 0 " , " 0 " , " 0 " , " 0 " ) ) ;
2020-12-30 14:56:37 +01:00
fromCalendar . add ( Calendar . MONTH , 1 ) ;
from = fromCalendar . getTime ( ) ;
}
}
2021-01-06 11:34:30 +01:00
public String getInstitutionName ( String repositoryIdentifier ) {
PreparedStatement st = null ;
Connection connection = null ;
ResultSet rs = null ;
String institutionName = " " ;
try {
connection = usageStatsDB . getConnection ( ) ;
st = connection . prepareStatement ( " select name from " + statsDB + " .datasource where id=? " ) ;
st . setString ( 1 , repositoryIdentifier ) ;
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
institutionName = rs . getString ( 1 ) ;
}
return institutionName ;
} catch ( Exception e ) {
log . error ( " Repository name failed: " , e ) ;
} finally {
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return " -1 " ;
}
2021-03-14 15:24:14 +01:00
public boolean checkServiceConnection ( ) {
Connection connection = null ;
try {
connection = usageStatsDB . getConnection ( ) ;
if ( connection ! = null ) {
return true ;
}
} catch ( Exception e ) {
log . info ( e ) ;
2021-03-16 06:56:18 +01:00
} finally {
2021-03-14 15:24:14 +01:00
DbUtils . closeQuietly ( connection ) ;
}
return false ;
}
public ArrayList buildMembersList ( ) {
PreparedStatement st = null ;
Connection connection = null ;
ResultSet rs = null ;
ArrayList membersList = new ArrayList ( ) ;
try {
connection = usageStatsDB . getConnection ( ) ;
st = connection . prepareStatement ( " select distinct us.repository_id, d.name, do.oid from " + statsDB + " .datasource d, "
+ statsDB + " .datasource_oids do, " + usagestatsImpalaDB + " .usage_stats us "
+ " where us.repository_id=d.id and us.repository_id=do.id and d.id=do.id and do.oid not like '%piwik%' order by do.oid asc " ) ;
rs = st . executeQuery ( ) ;
while ( rs . next ( ) ) {
ArrayList identifiers = new ArrayList ( ) ;
identifiers . add ( new SUSHI_Org_Identifiers ( " OpenAIRE " , rs . getString ( 1 ) ) ) ;
identifiers . add ( new SUSHI_Org_Identifiers ( " OpenDOAR " , rs . getString ( 3 ) ) ) ;
membersList . add ( new SUSHI_Consortium_Member_List ( null , null , rs . getString ( 2 ) , null , identifiers ) ) ;
}
} catch ( Exception e ) {
log . error ( " No member found " + e ) ;
2021-03-16 06:56:18 +01:00
} finally {
2021-03-14 15:24:14 +01:00
DbUtils . closeQuietly ( rs ) ;
DbUtils . closeQuietly ( st ) ;
DbUtils . closeQuietly ( connection ) ;
}
return membersList ;
}
2020-12-30 14:56:37 +01:00
}