spyros
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Moving variable declarations to a more appropriate place, adding drop table code

This commit is contained in:
Spyros Zoupanos 2020-09-04 19:49:07 +03:00
parent cf7b9c6db3
commit 5af2abbea5
3 changed files with 26 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikDownloadLogs.java
View File

@ -58,7 +58,6 @@ public class PiwikDownloadLogs {
try { try {
System.out.println("===> Connecting to: " + url); System.out.println("===> Connecting to: " + url);
URL website = new URL(url); URL website = new URL(url);
System.out.println("Connection url -----> " + url);
URLConnection connection = website.openConnection(); URLConnection connection = website.openConnection();
// connection.setRequestProperty ("Authorization", "Basic "+encoded); // connection.setRequestProperty ("Authorization", "Basic "+encoded);
@ -72,7 +71,7 @@ public class PiwikDownloadLogs {
} }
} }
System.out.println("response ====> " + response.toString()); // System.out.println("response ====> " + response.toString());
return response.toString(); return response.toString();
} catch (Exception e) { } catch (Exception e) {

18
dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
View File

@ -204,9 +204,15 @@ public class PiwikStatsDB {
// } // }
public void processRepositoryLog() throws Exception { public void processRepositoryLog() throws Exception {
Statement stmt = ConnectDB.getConnection().createStatement(); Statement stmt = ConnectDB.getConnection().createStatement();
ConnectDB.getConnection().setAutoCommit(false); ConnectDB.getConnection().setAutoCommit(false);
String drop_piwiklogtmp_json = "DROP TABLE IF EXISTS " +
ConnectDB.getUsageStatsDBSchema() +
".piwiklogtmp_json";
stmt.executeUpdate(drop_piwiklogtmp_json);
String stm_piwiklogtmp_json = "CREATE EXTERNAL TABLE IF NOT EXISTS " + String stm_piwiklogtmp_json = "CREATE EXTERNAL TABLE IF NOT EXISTS " +
ConnectDB.getUsageStatsDBSchema() + ConnectDB.getUsageStatsDBSchema() +
".piwiklogtmp_json(\n" + ".piwiklogtmp_json(\n" +
@ -229,16 +235,20 @@ public class PiwikStatsDB {
" >\n" + " >\n" +
")\n" + ")\n" +
"ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'\n" + "ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'\n" +
"LOCATION '/user/spyros/logs/usage_stats_logs/Repologs2/'\n" + "LOCATION '" + UsageStatsExporter.repoLogPath + "'\n" +
"TBLPROPERTIES (\"transactional\"=\"false\");\n" + "TBLPROPERTIES (\"transactional\"=\"false\")";
"";
stmt.executeUpdate(stm_piwiklogtmp_json); stmt.executeUpdate(stm_piwiklogtmp_json);
String drop_piwiklogtmp = "DROP TABLE IF EXISTS " +
ConnectDB.getUsageStatsDBSchema() +
".piwiklogtmp";
stmt.executeUpdate(drop_piwiklogtmp);
String stm_piwiklogtmp = "CREATE TABLE " + String stm_piwiklogtmp = "CREATE TABLE " +
ConnectDB.getUsageStatsDBSchema() + ConnectDB.getUsageStatsDBSchema() +
".piwiklogtmp (source BIGINT, id_Visit STRING, country STRING, action STRING, url STRING, " + ".piwiklogtmp (source BIGINT, id_Visit STRING, country STRING, action STRING, url STRING, " +
"entity_id STRING, source_item_type STRING, timestamp STRING, referrer_name STRING, agent STRING) " + "entity_id STRING, source_item_type STRING, timestamp STRING, referrer_name STRING, agent STRING) " +
"clustered by (source) into 100 buckets stored as orc tblproperties('transactional'='true');"; "clustered by (source) into 100 buckets stored as orc tblproperties('transactional'='true')";
stmt.executeUpdate(stm_piwiklogtmp); stmt.executeUpdate(stm_piwiklogtmp);
stmt.close(); stmt.close();

22
dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
View File

@ -10,6 +10,16 @@ public class UsageStatsExporter {
private Logger log = Logger.getLogger(this.getClass()); private Logger log = Logger.getLogger(this.getClass());
private Properties properties; private Properties properties;
static String matomoAuthToken = "703bd17d845acdaf795e01bb1e0895b9";
static String matomoBaseURL = "analytics.openaire.eu";
static String repoLogPath = "/user/spyros/logs/usage_stats_logs2/Repologs";
static String portalLogPath = "/user/spyros/logs/usage_stats_logs2/Portallogs/";
static String portalMatomoID = "109";
static String irusUKBaseURL = "https://irus.jisc.ac.uk/api/sushilite/v1_7/";
static String irusUKReportPath = "/user/spyros/logs/usage_stats_logs2/irusUKReports";
static String sarcsReportPath = "/user/spyros/logs/usage_stats_logs2/sarcReports";
public UsageStatsExporter(Properties properties) { public UsageStatsExporter(Properties properties) {
this.properties = properties; this.properties = properties;
} }
@ -25,16 +35,6 @@ public class UsageStatsExporter {
// String portalMatomoID = properties.getProperty("portal_MatomoID"); // String portalMatomoID = properties.getProperty("portal_MatomoID");
// String irusUKBaseURL = properties.getProperty("IRUS_UK_BaseUrl"); // String irusUKBaseURL = properties.getProperty("IRUS_UK_BaseUrl");
String matomoAuthToken = "703bd17d845acdaf795e01bb1e0895b9";
String matomoBaseURL = "analytics.openaire.eu";
String repoLogPath = "/user/spyros/logs/usage_stats_logs2/Repologs";
String portalLogPath = "/user/spyros/logs/usage_stats_logs2/Portallogs/";
String portalMatomoID = "109";
String irusUKBaseURL = "https://irus.jisc.ac.uk/api/sushilite/v1_7/";
String irusUKReportPath = "/user/spyros/logs/usage_stats_logs2/irusUKReports";
String sarcsReportPath = "/user/spyros/logs/usage_stats_logs2/sarcReports";
// connect to DB // connect to DB
ConnectDB.init(properties); ConnectDB.init(properties);
@ -44,7 +44,7 @@ public class UsageStatsExporter {
// // Download the statistics - The following 2 lines are not needed after the download - Commenting them out for // // Download the statistics - The following 2 lines are not needed after the download - Commenting them out for
// // the moment // // the moment
PiwikDownloadLogs piwd = new PiwikDownloadLogs(matomoBaseURL, matomoAuthToken); PiwikDownloadLogs piwd = new PiwikDownloadLogs(matomoBaseURL, matomoAuthToken);
piwd.GetOpenAIRELogs(repoLogPath, portalLogPath, portalMatomoID); // piwd.GetOpenAIRELogs(repoLogPath, portalLogPath, portalMatomoID);
// Create DB tables, insert/update statistics // Create DB tables, insert/update statistics
// String cRobotsUrl = properties.getProperty("COUNTER_robots_Url"); // String cRobotsUrl = properties.getProperty("COUNTER_robots_Url");