forked from D-Net/dnet-hadoop
lareferencia viewsStats & downloadsStats finished
This commit is contained in:
parent
65acece7c4
commit
1ceb363cb2
|
|
@ -112,9 +112,14 @@ public class LaReferenciaStats {
|
|||
// removeDoubleClicks();
|
||||
System.out.println("====> LaReferencia removed double clicks");
|
||||
log.info("LaReferencia removing double clicks done");
|
||||
viewsStats();
|
||||
// log.info("LaReferencia views done");
|
||||
// downloadsStats();
|
||||
|
||||
System.out.println("====> LaReferencia creating viewsStats");
|
||||
// viewsStats();
|
||||
System.out.println("====> LaReferencia created viewsStats");
|
||||
log.info("LaReferencia views done");
|
||||
System.out.println("====> LaReferencia creating downloadsStats");
|
||||
downloadsStats();
|
||||
System.out.println("====> LaReferencia created downloadsStats");
|
||||
// log.info("LaReferencia downloads done");
|
||||
// updateProdTables();
|
||||
// log.info("LaReferencia update productions tables done");
|
||||
|
|
@ -250,22 +255,39 @@ public class LaReferenciaStats {
|
|||
Statement stmt = ConnectDB.getConnection().createStatement();
|
||||
ConnectDB.getConnection().setAutoCommit(false);
|
||||
|
||||
// String sql = "CREATE OR REPLACE VIEW result_views_monthly AS SELECT entity_id AS id, COUNT(entity_id) as
|
||||
// views, extract('year' from timestamp::date) ||'/'|| LPAD(CAST(extract('month' from timestamp::date) AS
|
||||
// VARCHAR), 2, '0') AS month, source FROM lareferencialog where action='action' and (source_item_type='oaItem'
|
||||
// or source_item_type='repItem') group by id, month, source order by source, id, month;";
|
||||
String sql = "CREATE OR REPLACE VIEW la_result_views_monthly_tmp AS SELECT entity_id AS id, COUNT(entity_id) as views, SUM(CASE WHEN referrer_name LIKE '%openaire%' THEN 1 ELSE 0 END) AS openaire_referrer, extract('year' from timestamp::date) ||'/'|| LPAD(CAST(extract('month' from timestamp::date) AS VARCHAR), 2, '0') AS month, source FROM lareferencialogtmp where action='action' and (source_item_type='oaItem' or source_item_type='repItem') group by id, month, source order by source, id, month;";
|
||||
System.out.println("====> Creating la_result_views_monthly_tmp view");
|
||||
String sql = "CREATE OR REPLACE VIEW " + ConnectDB.getUsageStatsDBSchema() + ".la_result_views_monthly_tmp AS "
|
||||
+
|
||||
"SELECT entity_id AS id, COUNT(entity_id) as views, SUM(CASE WHEN referrer_name LIKE '%openaire%' " +
|
||||
"THEN 1 ELSE 0 END) AS openaire_referrer, " +
|
||||
"CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')) AS month, source " +
|
||||
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".lareferencialogtmp where action='action' and " +
|
||||
"(source_item_type='oaItem' or source_item_type='repItem') " +
|
||||
"GROUP BY entity_id, CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')), " +
|
||||
"source ORDER BY source, entity_id";
|
||||
stmt.executeUpdate(sql);
|
||||
System.out.println("====> Created la_result_views_monthly_tmp view");
|
||||
|
||||
// sql = "SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date,
|
||||
// max(views) AS count, max(openaire_referrer) AS openaire INTO views_stats FROM result_views_monthly p,
|
||||
// datasource d, result_oids ro where p.source!='5' AND p.source=d.piwik_id and p.id=ro.orid group by
|
||||
// repository_id, result_id, date ORDER BY repository_id, result_id, date;";
|
||||
sql = "CREATE TABLE IF NOT EXISTS la_views_stats_tmp AS SELECT 'LaReferencia'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(views) AS count, max(openaire_referrer) AS openaire FROM la_result_views_monthly_tmp p, public.datasource_oids d, public.result_oids ro where p.source=d.orid and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
|
||||
System.out.println("====> Dropping la_views_stats_tmp table");
|
||||
sql = "DROP TABLE IF EXISTS " +
|
||||
ConnectDB.getUsageStatsDBSchema() +
|
||||
".la_views_stats_tmp";
|
||||
stmt.executeUpdate(sql);
|
||||
System.out.println("====> Dropped la_views_stats_tmp table");
|
||||
|
||||
System.out.println("====> Creating la_views_stats_tmp table");
|
||||
sql = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".la_views_stats_tmp " +
|
||||
"AS SELECT 'LaReferencia' as source, d.id as repository_id, ro.id as result_id, month as date, " +
|
||||
"max(views) AS count, max(openaire_referrer) AS openaire " +
|
||||
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".la_result_views_monthly_tmp p, " +
|
||||
ConnectDB.getStatsDBSchema() + ".datasource_oids d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro " +
|
||||
"WHERE p.source=d.oid AND p.id=ro.oid " +
|
||||
"GROUP BY d.id, ro.id, month " +
|
||||
"ORDER BY d.id, ro.id, month";
|
||||
stmt.executeUpdate(sql);
|
||||
System.out.println("====> Created la_views_stats_tmp table");
|
||||
|
||||
stmt.close();
|
||||
ConnectDB.getConnection().commit();
|
||||
ConnectDB.getConnection().close();
|
||||
}
|
||||
|
||||
|
|
@ -274,24 +296,39 @@ public class LaReferenciaStats {
|
|||
Statement stmt = ConnectDB.getConnection().createStatement();
|
||||
ConnectDB.getConnection().setAutoCommit(false);
|
||||
|
||||
// String sql = "CREATE OR REPLACE VIEW result_downloads_monthly as select entity_id AS id, COUNT(entity_id) as
|
||||
// downloads, extract('year' from timestamp::date) ||'/'|| LPAD(CAST(extract('month' from timestamp::date) AS
|
||||
// VARCHAR), 2, '0') AS month, source FROM lareferencialog where action='download' and
|
||||
// (source_item_type='oaItem' or source_item_type='repItem') group by id, month, source order by source, id,
|
||||
// month;";
|
||||
String sql = "CREATE OR REPLACE VIEW la_result_downloads_monthly_tmp as select entity_id AS id, COUNT(entity_id) as downloads, SUM(CASE WHEN referrer_name LIKE '%openaire%' THEN 1 ELSE 0 END) AS openaire_referrer, extract('year' from timestamp::date) ||'/'|| LPAD(CAST(extract('month' from timestamp::date) AS VARCHAR), 2, '0') AS month, source FROM lareferencialogtmp where action='download' and (source_item_type='oaItem' or source_item_type='repItem') group by id, month, source order by source, id, month;";
|
||||
System.out.println("====> Creating la_result_downloads_monthly_tmp view");
|
||||
String sql = "CREATE OR REPLACE VIEW " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".la_result_downloads_monthly_tmp AS " +
|
||||
"SELECT entity_id AS id, COUNT(entity_id) as downloads, SUM(CASE WHEN referrer_name LIKE '%openaire%' " +
|
||||
"THEN 1 ELSE 0 END) AS openaire_referrer, " +
|
||||
"CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')) AS month, source " +
|
||||
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".lareferencialogtmp where action='download' and " +
|
||||
"(source_item_type='oaItem' or source_item_type='repItem') " +
|
||||
"GROUP BY entity_id, CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')), " +
|
||||
"source ORDER BY source, entity_id";
|
||||
stmt.executeUpdate(sql);
|
||||
System.out.println("====> Created la_result_downloads_monthly_tmp view");
|
||||
|
||||
// sql = "SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date,
|
||||
// max(downloads) AS count INTO downloads_stats FROM result_downloads_monthly p, datasource d, result_oids ro
|
||||
// where p.source!='5' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY
|
||||
// repository_id, result_id, date;";
|
||||
// sql = "SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(downloads) AS count, max(openaire_referrer) AS openaire INTO downloads_stats FROM result_downloads_monthly p, datasource d, result_oids ro where p.source!='5' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
|
||||
sql = "CREATE TABLE IF NOT EXISTS la_downloads_stats_tmp AS SELECT 'LaReferencia'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(downloads) AS count, max(openaire_referrer) AS openaire FROM la_result_downloads_monthly_tmp p, public.datasource_oids d, public.result_oids ro where p.source=d.orid and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
|
||||
System.out.println("====> Dropping la_views_stats_tmp table");
|
||||
sql = "DROP TABLE IF EXISTS " +
|
||||
ConnectDB.getUsageStatsDBSchema() +
|
||||
".la_views_stats_tmp";
|
||||
stmt.executeUpdate(sql);
|
||||
System.out.println("====> Dropped la_views_stats_tmp table");
|
||||
|
||||
System.out.println("====> Creating la_downloads_stats_tmp table");
|
||||
sql = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".la_downloads_stats_tmp " +
|
||||
"AS SELECT 'LaReferencia' as source, d.id as repository_id, ro.id as result_id, month as date, " +
|
||||
"max(downloads) AS count, max(openaire_referrer) AS openaire " +
|
||||
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".la_result_views_monthly_tmp p, " +
|
||||
ConnectDB.getStatsDBSchema() + ".datasource_oids d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro " +
|
||||
"WHERE p.source=d.oid AND p.id=ro.oid " +
|
||||
"GROUP BY d.id, ro.id, month " +
|
||||
"ORDER BY d.id, ro.id, month";
|
||||
stmt.executeUpdate(sql);
|
||||
System.out.println("====> Created la_downloads_stats_tmp table");
|
||||
|
||||
stmt.close();
|
||||
ConnectDB.getConnection().commit();
|
||||
ConnectDB.getConnection().close();
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue