diff --git a/dhp-workflows/dhp-indicators/nb-configuration.xml b/dhp-workflows/dhp-indicators/nb-configuration.xml
deleted file mode 100644
index a65c4514a..000000000
--- a/dhp-workflows/dhp-indicators/nb-configuration.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-
-
-
- JDK_1.8
-
-
diff --git a/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/config-default.xml b/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/config-default.xml
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/scripts/createIndicatorsTablesSprint1.sql b/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/scripts/createIndicatorsTablesSprint1.sql
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/scripts/createIndicatorsTablesSprint2.sql b/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/scripts/createIndicatorsTablesSprint2.sql
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/scripts/indicatorsSprint1.sh b/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/scripts/indicatorsSprint1.sh
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/scripts/indicatorsSprint2.sh b/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/scripts/indicatorsSprint2.sh
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/workflow.xml b/dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/workflow.xml
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-datasets-stats-update/nb-configuration.xml b/dhp-workflows/dhp-usage-datasets-stats-update/nb-configuration.xml
deleted file mode 100644
index a65c4514a..000000000
--- a/dhp-workflows/dhp-usage-datasets-stats-update/nb-configuration.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-
-
-
- JDK_1.8
-
-
diff --git a/dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/ConnectDB.java b/dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/ConnectDB.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/DatasetsStatsDB.java b/dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/DatasetsStatsDB.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/DownloadReportsListFromDatacite.java b/dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/DownloadReportsListFromDatacite.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/ExecuteWorkflow.java b/dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/ExecuteWorkflow.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/ReadReportsListFromDatacite.java b/dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/ReadReportsListFromDatacite.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-datasets-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/datasetsusagestats/export/datasets_usagestats_parameters.json b/dhp-workflows/dhp-usage-datasets-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/datasetsusagestats/export/datasets_usagestats_parameters.json
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-datasets-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/datasetsusagestats/oozie_app/config-default.xml b/dhp-workflows/dhp-usage-datasets-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/datasetsusagestats/oozie_app/config-default.xml
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-datasets-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/datasetsusagestats/oozie_app/workflow.xml b/dhp-workflows/dhp-usage-datasets-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/datasetsusagestats/oozie_app/workflow.xml
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/pom.xml b/dhp-workflows/dhp-usage-raw-data-update-beta/pom.xml
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/ConnectDB.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/ConnectDB.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/ExecuteWorkflow.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/ExecuteWorkflow.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/IrusStats.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/IrusStats.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/LaReferenciaDownloadLogs.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/LaReferenciaDownloadLogs.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/LaReferenciaStats.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/LaReferenciaStats.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/PiwikDownloadLogs.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/PiwikDownloadLogs.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/PiwikDownloadLogs_B2SHARE.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/PiwikDownloadLogs_B2SHARE.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/PiwikStatsDB.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/PiwikStatsDB_B2SHARE.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/PiwikStatsDB_B2SHARE.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/ReadCounterRobotsList.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/ReadCounterRobotsList.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/SarcStats.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/SarcStats.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/java/eu/dnetlib/oa/graph/usagerawdatabeta/export/UsageStatsExporter.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdatabeta/export/usagerawdata_parameters.json b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdatabeta/export/usagerawdata_parameters.json
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdatabeta/oozie_app/config-default.xml b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdatabeta/oozie_app/config-default.xml
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdatabeta/oozie_app/workflow.xml b/dhp-workflows/dhp-usage-raw-data-update-beta/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdatabeta/oozie_app/workflow.xml
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/ExecuteWorkflow.java b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/ExecuteWorkflow.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/IrusStats.java b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/IrusStats.java
old mode 100644
new mode 100755
index 7ec5b0fca..38ed3ac56
--- a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/IrusStats.java
+++ b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/IrusStats.java
@@ -34,8 +34,6 @@ public class IrusStats {
public IrusStats(String irusUKURL) throws Exception {
this.irusUKURL = irusUKURL;
- // The following may not be needed - It will be created when JSON tables are created
-// createTmpTables();
}
public void reCreateLogDirs() throws Exception {
diff --git a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/LaReferenciaDownloadLogs.java b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/LaReferenciaDownloadLogs.java
old mode 100644
new mode 100755
index 904290af8..1445406a8
--- a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/LaReferenciaDownloadLogs.java
+++ b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/LaReferenciaDownloadLogs.java
@@ -71,16 +71,6 @@ public class LaReferenciaDownloadLogs {
+ "stored as orc tblproperties('transactional'='true')";
stmt.executeUpdate(sqlCreateTableLareferenciaLog);
logger.info("Created LaReferencia tables");
-// String sqlcreateRuleLaReferenciaLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS "
-// + " ON INSERT TO lareferencialog "
-// + " WHERE (EXISTS ( SELECT lareferencialog.matomoid, lareferencialog.source, lareferencialog.id_visit,"
-// + "lareferencialog.action, lareferencialog.\"timestamp\", lareferencialog.entity_id "
-// + "FROM lareferencialog "
-// + "WHERE lareferencialog.matomoid=new.matomoid AND lareferencialog.source = new.source AND lareferencialog.id_visit = new.id_visit AND lareferencialog.action = new.action AND lareferencialog.entity_id = new.entity_id AND lareferencialog.\"timestamp\" = new.\"timestamp\")) DO INSTEAD NOTHING;";
-// String sqlCreateRuleIndexLaReferenciaLog = "create index if not exists lareferencialog_rule on lareferencialog(matomoid, source, id_visit, action, entity_id, \"timestamp\");";
-// stmt.executeUpdate(sqlcreateRuleLaReferenciaLog);
-// stmt.executeUpdate(sqlCreateRuleIndexLaReferenciaLog);
-
stmt.close();
ConnectDB.getHiveConnection().close();
logger.info("Lareferencia Tables Created");
@@ -92,29 +82,6 @@ public class LaReferenciaDownloadLogs {
}
}
-// private void createTmpTables() throws Exception {
-//
-// try {
-// Statement stmt = ConnectDB.getConnection().createStatement();
-// String sqlCreateTmpTableLaReferenciaLog = "CREATE TABLE IF NOT EXISTS lareferencialogtmp(matomoid INTEGER, source TEXT, id_visit TEXT, country TEXT, action TEXT, url TEXT, entity_id TEXT, source_item_type TEXT, timestamp TEXT, referrer_name TEXT, agent TEXT, PRIMARY KEY(source, id_visit, action, timestamp, entity_id));";
-// String sqlcreateTmpRuleLaReferenciaLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS "
-// + " ON INSERT TO lareferencialogtmp "
-// + " WHERE (EXISTS ( SELECT lareferencialogtmp.matomoid, lareferencialogtmp.source, lareferencialogtmp.id_visit,"
-// + "lareferencialogtmp.action, lareferencialogtmp.\"timestamp\", lareferencialogtmp.entity_id "
-// + "FROM lareferencialogtmp "
-// + "WHERE lareferencialogtmp.matomoid=new.matomoid AND lareferencialogtmp.source = new.source AND lareferencialogtmp.id_visit = new.id_visit AND lareferencialogtmp.action = new.action AND lareferencialogtmp.entity_id = new.entity_id AND lareferencialogtmp.\"timestamp\" = new.\"timestamp\")) DO INSTEAD NOTHING;";
-// stmt.executeUpdate(sqlCreateTmpTableLaReferenciaLog);
-// stmt.executeUpdate(sqlcreateTmpRuleLaReferenciaLog);
-//
-// stmt.close();
-// log.info("Lareferencia Tmp Tables Created");
-//
-// } catch (Exception e) {
-// log.error("Failed to create tmptables: " + e);
-// throw new Exception("Failed to create tmp tables: " + e.toString(), e);
-// // System.exit(0);
-// }
-// }
private String getPiwikLogUrl() {
return piwikUrl + "/";
}
diff --git a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikDownloadLogs.java b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikDownloadLogs.java
old mode 100644
new mode 100755
index b4f8cdf2c..38394288d
--- a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikDownloadLogs.java
+++ b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikDownloadLogs.java
@@ -217,6 +217,7 @@ public class PiwikDownloadLogs {
while (rs.next()) {
piwikIdToVisit.add(rs.getInt(1));
}
+ piwikIdToVisit.add(548);
logger.info("Found the following piwikIds for download: " + piwikIdToVisit);
if (ExecuteWorkflow.numberOfPiwikIdsToDownload > 0
diff --git a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikDownloadLogs_B2SHARE.java b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikDownloadLogs_B2SHARE.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikStatsDB.java
old mode 100644
new mode 100755
index b07322beb..9fa008f4c
--- a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikStatsDB.java
+++ b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikStatsDB.java
@@ -118,12 +118,6 @@ public class PiwikStatsDB {
+ "into 100 buckets stored as orc tblproperties('transactional'='true')";
stmt.executeUpdate(sqlCreateTablePiwikLog);
-// String dropT = "TRUNCATE TABLE "
-// + ConnectDB.getUsageStatsDBSchema()
-// + ".piwiklog ";
-// stmt.executeUpdate(dropT);
-// logger.info("truncated piwiklog");
-
/////////////////////////////////////////
// Rule for duplicate inserts @ piwiklog
/////////////////////////////////////////
diff --git a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikStatsDB_B2SHARE.java b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikStatsDB_B2SHARE.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/UsageStatsExporter.java
old mode 100644
new mode 100755
index 2f10e4d2b..33fad50bd
--- a/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/UsageStatsExporter.java
+++ b/dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/UsageStatsExporter.java
@@ -2,7 +2,6 @@
package eu.dnetlib.oa.graph.usagerawdata.export;
import java.io.IOException;
-import java.sql.SQLException;
import java.sql.Statement;
import org.apache.hadoop.conf.Configuration;
diff --git a/dhp-workflows/dhp-usage-raw-data-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdata/export/usagerawdata_parameters.json b/dhp-workflows/dhp-usage-raw-data-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdata/export/usagerawdata_parameters.json
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-raw-data-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdata/oozie_app/workflow.xml b/dhp-workflows/dhp-usage-raw-data-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdata/oozie_app/workflow.xml
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-stats-build/src/main/java/eu/dnetlib/oa/graph/usagestatsbuild/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-stats-build/src/main/java/eu/dnetlib/oa/graph/usagestatsbuild/export/PiwikStatsDB.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-stats-build/src/main/java/eu/dnetlib/oa/graph/usagestatsbuild/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-build/src/main/java/eu/dnetlib/oa/graph/usagestatsbuild/export/UsageStatsExporter.java
old mode 100644
new mode 100755
diff --git a/dhp-workflows/dhp-usage-stats-build/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestatsbuild/oozie_app/workflow.xml b/dhp-workflows/dhp-usage-stats-build/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestatsbuild/oozie_app/workflow.xml
old mode 100644
new mode 100755