From c0b509abfbc7de7fed2bfcfdfde8fd8b5f80eba7 Mon Sep 17 00:00:00 2001 From: Spyros Zoupanos Date: Sat, 9 May 2020 13:20:41 +0300 Subject: [PATCH] Simple java action added. Simple java connection to hive db + basic statements added --- .../dhp-java-action-dbconnection/pom.xml | 35 +++++++ .../oa/graph/usagestats/export/Simple.java | 99 +++++++++++++++++++ .../usagestats}/oozie_app/config-default.xml | 4 + .../graph/usagestats/oozie_app/workflow.xml | 48 +++++++++ dhp-workflows/dhp-java-action-simple/pom.xml | 13 +++ .../oa/graph/usagestats/export/Simple.java | 12 +++ .../usagestats/oozie_app/config-default.xml | 34 +++++++ .../graph/usagestats/oozie_app/workflow.xml | 48 +++++++++ dhp-workflows/dhp-usage-stats-update/pom.xml | 4 +- .../export/ClouderaImpalaJdbcExample.java | 60 +++++++++++ .../export/ConnectDB.java | 0 .../export/ExecuteWorkflow.java | 0 .../export/IrusStats.java | 0 .../export/PiwikDownloadLogs.java | 0 .../export/PiwikStatsDB.java | 0 .../export/ReadCounterRobotsList.java | 0 .../export/SarcStats.java | 0 .../export/UsageStatsExporter.java | 0 .../export/index.html | 48 ++++----- .../usagestats/oozie_app/config-default.xml | 34 +++++++ .../oozie_app/workflow.xml | 47 ++++++++- dhp-workflows/pom.xml | 2 + 22 files changed, 460 insertions(+), 28 deletions(-) create mode 100644 dhp-workflows/dhp-java-action-dbconnection/pom.xml create mode 100755 dhp-workflows/dhp-java-action-dbconnection/src/main/java/eu/dnetlib/oa/graph/usagestats/export/Simple.java rename dhp-workflows/{dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usage-stats => dhp-java-action-dbconnection/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats}/oozie_app/config-default.xml (89%) create mode 100644 dhp-workflows/dhp-java-action-dbconnection/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-java-action-simple/pom.xml create mode 100755 dhp-workflows/dhp-java-action-simple/src/main/java/eu/dnetlib/oa/graph/usagestats/export/Simple.java create mode 100644 dhp-workflows/dhp-java-action-simple/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-java-action-simple/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml create mode 100755 dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ClouderaImpalaJdbcExample.java rename dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/{usage-stats => usagestats}/export/ConnectDB.java (100%) rename dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/{usage-stats => usagestats}/export/ExecuteWorkflow.java (100%) rename dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/{usage-stats => usagestats}/export/IrusStats.java (100%) rename dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/{usage-stats => usagestats}/export/PiwikDownloadLogs.java (100%) rename dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/{usage-stats => usagestats}/export/PiwikStatsDB.java (100%) rename dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/{usage-stats => usagestats}/export/ReadCounterRobotsList.java (100%) rename dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/{usage-stats => usagestats}/export/SarcStats.java (100%) rename dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/{usage-stats => usagestats}/export/UsageStatsExporter.java (100%) rename dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/{usage-stats => usagestats}/export/index.html (60%) create mode 100644 dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml rename dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/{usage-stats => usagestats}/oozie_app/workflow.xml (59%) diff --git a/dhp-workflows/dhp-java-action-dbconnection/pom.xml b/dhp-workflows/dhp-java-action-dbconnection/pom.xml new file mode 100644 index 0000000000..826e796425 --- /dev/null +++ b/dhp-workflows/dhp-java-action-dbconnection/pom.xml @@ -0,0 +1,35 @@ + + + + + + dhp-workflows + eu.dnetlib.dhp + 1.1.7-SNAPSHOT + + eu.dnetlib + 4.0.0 + dhp-java-action-dbconnection + + + UTF-8 + UTF-8 + 0.13.1-cdh5.2.1 + 2.5.0-cdh5.2.1 + + + + + org.apache.hive + hive-jdbc + ${cdh.hive.version} + + + + + org.apache.hadoop + hadoop-common + ${cdh.hadoop.version} + + + diff --git a/dhp-workflows/dhp-java-action-dbconnection/src/main/java/eu/dnetlib/oa/graph/usagestats/export/Simple.java b/dhp-workflows/dhp-java-action-dbconnection/src/main/java/eu/dnetlib/oa/graph/usagestats/export/Simple.java new file mode 100755 index 0000000000..755acd395c --- /dev/null +++ b/dhp-workflows/dhp-java-action-dbconnection/src/main/java/eu/dnetlib/oa/graph/usagestats/export/Simple.java @@ -0,0 +1,99 @@ + +package eu.dnetlib.oa.graph.usagestats.export; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +public class Simple { + + public static void main(final String[] args) throws Exception { + + System.out.println("============================================="); + System.out.println("Staring"); + System.out.println("============================================="); + + // Connection string that works OK + // String connectionUrl = + // "jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/stats_wf_db_galexiou_oozie_beta;UID=spyros;PWD=XXXXXX;UseNativeQuery=1"; + + // The following connectio string also works + String connectionUrl = "jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1"; + String jdbcDriverName = "org.apache.hive.jdbc.HiveDriver"; + + System.out.println("\n============================================="); + System.out.println("Cloudera Impala JDBC Example"); + System.out.println("Using Connection URL: " + connectionUrl); + System.out.println("USing JDBC Driver " + jdbcDriverName); + + Connection con = null; + + try { + System.out.println("Step 1"); + Class.forName(jdbcDriverName); + System.out.println("Step 2"); + con = DriverManager.getConnection(connectionUrl); + System.out.println("Step 3"); + Statement stmt = con.createStatement(); + System.out.println("Step 4"); + ResultSet rs = stmt.executeQuery("select personid, lastname from test_db_spyros.persons"); + + System.out.println("\n== Begin Query Results ======================"); + + // print the results to the console + while (rs.next()) { + // the example query returns one String column + System.out.println(rs.getString(1) + " | " + rs.getString(2)); + } + + System.out.println("== End Query Results =======================\n\n"); + + // Drop table if exists + stmt.execute("DROP TABLE IF EXISTS test_db_spyros.Persons2"); + + // Create table + stmt + .execute( + "CREATE TABLE test_db_spyros.Persons2 (PersonID int, LastName varchar(255), FirstName varchar(255))"); + + // Insert + stmt + .execute( + "INSERT INTO test_db_spyros.persons2 (personid, lastname, firstname) VALUES ('1', 'ZoupZoup', 'Spyros')"); + + // Select the inserted values + rs = stmt.executeQuery("select personid, lastname, firstname from test_db_spyros.persons2"); + while (rs.next()) { + // the example query returns one String column + System.out.println(rs.getString(1) + " | " + rs.getString(2) + " | " + rs.getString(3)); + } + + } catch (SQLException e) { + System.out.println("Ex 1"); + e.printStackTrace(); + System.out.println(e.getMessage()); + } catch (Exception e) { + System.out.println("Ex 2"); + e.printStackTrace(); + System.out.println(e.getMessage()); + } finally { + try { + con.close(); + } catch (Exception e) { + // swallow + System.out.println("Ex 3"); + e.printStackTrace(); + System.out.println(e.getMessage()); + } + } + + System.out.println("============================================="); + System.out.println("Ending"); + System.out.println("============================================="); + } +} diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usage-stats/oozie_app/config-default.xml b/dhp-workflows/dhp-java-action-dbconnection/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml similarity index 89% rename from dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usage-stats/oozie_app/config-default.xml rename to dhp-workflows/dhp-java-action-dbconnection/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml index ba7002cffc..d369804b63 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usage-stats/oozie_app/config-default.xml +++ b/dhp-workflows/dhp-java-action-dbconnection/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml @@ -27,4 +27,8 @@ oozie.wf.workflow.notification.url {serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status + + oozie.use.system.libpath + true + diff --git a/dhp-workflows/dhp-java-action-dbconnection/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml b/dhp-workflows/dhp-java-action-dbconnection/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml new file mode 100644 index 0000000000..daae758444 --- /dev/null +++ b/dhp-workflows/dhp-java-action-dbconnection/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml @@ -0,0 +1,48 @@ + + + + hiveMetastoreUris + hive server metastore URIs + + + hiveJdbcUrl + hive server jdbc url + + + + + ${jobTracker} + ${nameNode} + + + hive.metastore.uris + ${hiveMetastoreUris} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + eu.dnetlib.oa.graph.usagestats.export.Simple + + + + + + + + diff --git a/dhp-workflows/dhp-java-action-simple/pom.xml b/dhp-workflows/dhp-java-action-simple/pom.xml new file mode 100644 index 0000000000..3e0f75e45f --- /dev/null +++ b/dhp-workflows/dhp-java-action-simple/pom.xml @@ -0,0 +1,13 @@ + + + + + + dhp-workflows + eu.dnetlib.dhp + 1.1.7-SNAPSHOT + + eu.dnetlib + 4.0.0 + dhp-java-action-simple + diff --git a/dhp-workflows/dhp-java-action-simple/src/main/java/eu/dnetlib/oa/graph/usagestats/export/Simple.java b/dhp-workflows/dhp-java-action-simple/src/main/java/eu/dnetlib/oa/graph/usagestats/export/Simple.java new file mode 100755 index 0000000000..65fe8f7b8b --- /dev/null +++ b/dhp-workflows/dhp-java-action-simple/src/main/java/eu/dnetlib/oa/graph/usagestats/export/Simple.java @@ -0,0 +1,12 @@ + +package eu.dnetlib.oa.graph.usagestats.export; + +public class Simple { + + public static void main(final String[] args) throws Exception { + + System.out.println("============================================="); + System.out.println("Hello world"); + System.out.println("============================================="); + } +} diff --git a/dhp-workflows/dhp-java-action-simple/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml b/dhp-workflows/dhp-java-action-simple/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml new file mode 100644 index 0000000000..d369804b63 --- /dev/null +++ b/dhp-workflows/dhp-java-action-simple/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml @@ -0,0 +1,34 @@ + + + jobTracker + ${jobTracker} + + + nameNode + ${nameNode} + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hiveMetastoreUris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + hiveJdbcUrl + jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000 + + + oozie.wf.workflow.notification.url + {serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status + + + oozie.use.system.libpath + true + + diff --git a/dhp-workflows/dhp-java-action-simple/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml b/dhp-workflows/dhp-java-action-simple/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml new file mode 100644 index 0000000000..5bbf659b16 --- /dev/null +++ b/dhp-workflows/dhp-java-action-simple/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml @@ -0,0 +1,48 @@ + + + + hiveMetastoreUris + hive server metastore URIs + + + hiveJdbcUrl + hive server jdbc url + + + + + ${jobTracker} + ${nameNode} + + + hive.metastore.uris + ${hiveMetastoreUris} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + eu.dnetlib.oa.graph.usagestats.export.Simple + + + + + + + + diff --git a/dhp-workflows/dhp-usage-stats-update/pom.xml b/dhp-workflows/dhp-usage-stats-update/pom.xml index 7c07c8e1f8..f6f56d6d87 100644 --- a/dhp-workflows/dhp-usage-stats-update/pom.xml +++ b/dhp-workflows/dhp-usage-stats-update/pom.xml @@ -50,8 +50,8 @@ - + diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ClouderaImpalaJdbcExample.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ClouderaImpalaJdbcExample.java new file mode 100755 index 0000000000..e4c17b868e --- /dev/null +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ClouderaImpalaJdbcExample.java @@ -0,0 +1,60 @@ + +package eu.dnetlib.oa.graph.usagestats.export; + +public class ClouderaImpalaJdbcExample { + + public static void main(String[] args) { + + System.out.println("\n============================================="); + +//// loadConfiguration(); +// +//// String sqlStatement = args[0]; +// String sqlStatement = "select count(*) from fundref;"; +// connectionUrl = "jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/stats_wf_db_galexiou_oozie_beta;UID=spyros;PWD=RU78N9sqQndnH3SQ;UseNativeQuery=1"; +//// connectionUrl = "jdbc:hive2://172.16.10.74:21050/emay_data;auth=noSasl"; +//// "jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000" +//// url=jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/stats_wf_db_galexiou_oozie_beta;UID=antonis.lempesis;PWD=XXXXX;UseNativeQuery=1 +// jdbcDriverName = "org.apache.hive.jdbc.HiveDriver"; +// +// System.out.println("\n============================================="); +// System.out.println("Cloudera Impala JDBC Example"); +// System.out.println("Using Connection URL: " + connectionUrl); +// System.out.println("USing JDBC Driver " + jdbcDriverName); +// System.out.println("Running Query: " + sqlStatement); +// +// Connection con = null; +// +// try { +// +// Class.forName(jdbcDriverName); +// +// con = DriverManager.getConnection(connectionUrl); +// +// Statement stmt = con.createStatement(); +// +// ResultSet rs = stmt.executeQuery(sqlStatement); +// +// System.out.println("\n== Begin Query Results ======================"); +// +// // print the results to the console +// while (rs.next()) { +// // the example query returns one String column +// System.out.println(rs.getString(1)); +// } +// +// System.out.println("== End Query Results =======================\n\n"); +// +// } catch (SQLException e) { +// e.printStackTrace(); +// } catch (Exception e) { +// e.printStackTrace(); +// } finally { +// try { +// con.close(); +// } catch (Exception e) { +// // swallow +// } +// } + } +} diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/ConnectDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java similarity index 100% rename from dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/ConnectDB.java rename to dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/ExecuteWorkflow.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java similarity index 100% rename from dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/ExecuteWorkflow.java rename to dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/IrusStats.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/IrusStats.java similarity index 100% rename from dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/IrusStats.java rename to dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/IrusStats.java diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/PiwikDownloadLogs.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikDownloadLogs.java similarity index 100% rename from dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/PiwikDownloadLogs.java rename to dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikDownloadLogs.java diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java similarity index 100% rename from dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/PiwikStatsDB.java rename to dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/ReadCounterRobotsList.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ReadCounterRobotsList.java similarity index 100% rename from dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/ReadCounterRobotsList.java rename to dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ReadCounterRobotsList.java diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/SarcStats.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java similarity index 100% rename from dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/SarcStats.java rename to dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java similarity index 100% rename from dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/UsageStatsExporter.java rename to dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/index.html b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/index.html similarity index 60% rename from dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/index.html rename to dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/index.html index 9eac2342ff..d8741bc58d 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usage-stats/export/index.html +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/index.html @@ -1,43 +1,43 @@ - - Revision 58415: /dnet45/modules/dnet-openaire-usage-stats-export-wf/trunk/dnet-openaire-usage-stats-export/src/main/java/eu/dnetlib/usagestats/export + + Revision 58415: /dnet45/modules/dnet-openaire-usage-stats-export-wf/trunk/dnet-openaire-usage-stats-export/src/main/java/eu/dnetlib/usagestats/export

Revision 58415: /dnet45/modules/dnet-openaire-usage-stats-export-wf/trunk/dnet-openaire-usage-stats-export/src/main/java/eu/dnetlib/usagestats/export

-
+
- Powered by + Powered by Subversion - version 1.4.4 (r25188). - + version 1.4.4 (r25188). + \ No newline at end of file diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml new file mode 100644 index 0000000000..d369804b63 --- /dev/null +++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml @@ -0,0 +1,34 @@ + + + jobTracker + ${jobTracker} + + + nameNode + ${nameNode} + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hiveMetastoreUris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + hiveJdbcUrl + jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000 + + + oozie.wf.workflow.notification.url + {serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status + + + oozie.use.system.libpath + true + + diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usage-stats/oozie_app/workflow.xml b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml similarity index 59% rename from dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usage-stats/oozie_app/workflow.xml rename to dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml index 70d4dcffcf..193fe41b42 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usage-stats/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml @@ -31,15 +31,58 @@ hive.metastore.uris ${hiveMetastoreUris} + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + - + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + mapred.queue.name + default + + + eu.dnetlib.oa.graph.usagestats.export.ClouderaImpalaJdbcExample + + + + + + + + + + + + + eu.dnetlib.dhp.oa.graph.raw.MigrateDbEntitiesApplication + --hdfsPath${contentPath}/db_claims + --postgresUrl${postgresURL} + --postgresUser${postgresUser} + --postgresPassword${postgresPassword} + --actionclaims + + + + + ${jobTracker} @@ -50,7 +93,7 @@ ${queueName} - eu.dnetlib.oa.graph.usage-stats.export.UsageStatsExporter + eu.dnetlib.oa.graph.usagestats.export.UsageStatsExporter org.apache.oozie.test.MyTest ${outputFileName} diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index ca82cf1faf..1e93fbd9c5 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -27,6 +27,8 @@ dhp-graph-provision-scholexplorer dhp-stats-update dhp-usage-stats-update + dhp-java-action-simple + dhp-java-action-dbconnection/ dhp-broker-events