Commit 13/12/2022

dhp-workflows/dhp-indicators/nb-configuration.xml

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project-shared-configuration>
-    <!--
-This file contains additional configuration written by modules in the NetBeans IDE.
-The configuration is intended to be shared among all the users of project and
-therefore it is assumed to be part of version control checkout.
-Without this configuration present, some functionality in the IDE may be limited or fail altogether.
--->
-    <properties xmlns="http://www.netbeans.org/ns/maven-properties-data/1">
-        <!--
-Properties that influence various parts of the IDE, especially code formatting and the like. 
-You can copy and paste the single properties, into the pom.xml file and the IDE will pick them up.
-That way multiple projects can share the same settings (useful for formatting rules for example).
-Any value defined here will override the pom.xml file value but is only applicable to the current project.
--->
-        <netbeans.hint.jdkPlatform>JDK_1.8</netbeans.hint.jdkPlatform>
-    </properties>
-</project-shared-configuration>

dhp-workflows/dhp-usage-datasets-stats-update/nb-configuration.xml

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project-shared-configuration>
-    <!--
-This file contains additional configuration written by modules in the NetBeans IDE.
-The configuration is intended to be shared among all the users of project and
-therefore it is assumed to be part of version control checkout.
-Without this configuration present, some functionality in the IDE may be limited or fail altogether.
--->
-    <properties xmlns="http://www.netbeans.org/ns/maven-properties-data/1">
-        <!--
-Properties that influence various parts of the IDE, especially code formatting and the like. 
-You can copy and paste the single properties, into the pom.xml file and the IDE will pick them up.
-That way multiple projects can share the same settings (useful for formatting rules for example).
-Any value defined here will override the pom.xml file value but is only applicable to the current project.
--->
-        <netbeans.hint.jdkPlatform>JDK_1.8</netbeans.hint.jdkPlatform>
-    </properties>
-</project-shared-configuration>

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/IrusStats.java

@@ -34,8 +34,6 @@ public class IrusStats {
 
 	public IrusStats(String irusUKURL) throws Exception {
 		this.irusUKURL = irusUKURL;
-		// The following may not be needed - It will be created when JSON tables are created
-//		createTmpTables();
 	}
 
 	public void reCreateLogDirs() throws Exception {

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/LaReferenciaDownloadLogs.java

@@ -71,16 +71,6 @@ public class LaReferenciaDownloadLogs {
 				+ "stored as orc tblproperties('transactional'='true')";
 			stmt.executeUpdate(sqlCreateTableLareferenciaLog);
 			logger.info("Created LaReferencia tables");
-//            String sqlcreateRuleLaReferenciaLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS "
-//                    + " ON INSERT TO lareferencialog "
-//                    + " WHERE (EXISTS ( SELECT lareferencialog.matomoid, lareferencialog.source, lareferencialog.id_visit,"
-//                    + "lareferencialog.action, lareferencialog.\"timestamp\", lareferencialog.entity_id "
-//                    + "FROM lareferencialog "
-//                    + "WHERE lareferencialog.matomoid=new.matomoid AND lareferencialog.source = new.source AND lareferencialog.id_visit = new.id_visit AND lareferencialog.action = new.action AND lareferencialog.entity_id = new.entity_id AND lareferencialog.\"timestamp\" = new.\"timestamp\")) DO INSTEAD NOTHING;";
-//            String sqlCreateRuleIndexLaReferenciaLog = "create index if not exists lareferencialog_rule on lareferencialog(matomoid, source, id_visit, action, entity_id, \"timestamp\");";
-//            stmt.executeUpdate(sqlcreateRuleLaReferenciaLog);
-//            stmt.executeUpdate(sqlCreateRuleIndexLaReferenciaLog);
-
 			stmt.close();
 			ConnectDB.getHiveConnection().close();
 			logger.info("Lareferencia Tables Created");
@@ -92,29 +82,6 @@ public class LaReferenciaDownloadLogs {
 		}
 	}
 
-//	private void createTmpTables() throws Exception {
-//
-//		try {
-//			Statement stmt = ConnectDB.getConnection().createStatement();
-//			String sqlCreateTmpTableLaReferenciaLog = "CREATE TABLE IF NOT EXISTS lareferencialogtmp(matomoid INTEGER, source TEXT, id_visit TEXT, country TEXT, action TEXT, url TEXT, entity_id TEXT, source_item_type TEXT, timestamp TEXT, referrer_name TEXT, agent TEXT, PRIMARY KEY(source, id_visit, action, timestamp, entity_id));";
-//			String sqlcreateTmpRuleLaReferenciaLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS "
-//				+ " ON INSERT TO lareferencialogtmp "
-//				+ " WHERE (EXISTS ( SELECT lareferencialogtmp.matomoid, lareferencialogtmp.source, lareferencialogtmp.id_visit,"
-//				+ "lareferencialogtmp.action, lareferencialogtmp.\"timestamp\", lareferencialogtmp.entity_id "
-//				+ "FROM lareferencialogtmp "
-//				+ "WHERE lareferencialogtmp.matomoid=new.matomoid AND lareferencialogtmp.source = new.source AND lareferencialogtmp.id_visit = new.id_visit AND lareferencialogtmp.action = new.action AND lareferencialogtmp.entity_id = new.entity_id AND lareferencialogtmp.\"timestamp\" = new.\"timestamp\")) DO INSTEAD NOTHING;";
-//			stmt.executeUpdate(sqlCreateTmpTableLaReferenciaLog);
-//			stmt.executeUpdate(sqlcreateTmpRuleLaReferenciaLog);
-//
-//			stmt.close();
-//			log.info("Lareferencia Tmp Tables Created");
-//
-//		} catch (Exception e) {
-//			log.error("Failed to create tmptables: " + e);
-//			throw new Exception("Failed to create tmp tables: " + e.toString(), e);
-//			// System.exit(0);
-//		}
-//	}
 	private String getPiwikLogUrl() {
 		return piwikUrl + "/";
 	}

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikDownloadLogs.java

@@ -217,6 +217,7 @@ public class PiwikDownloadLogs {
 		while (rs.next()) {
 			piwikIdToVisit.add(rs.getInt(1));
 		}
+		piwikIdToVisit.add(548);
 		logger.info("Found the following piwikIds for download: " + piwikIdToVisit);
 
 		if (ExecuteWorkflow.numberOfPiwikIdsToDownload > 0

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikStatsDB.java

@@ -118,12 +118,6 @@ public class PiwikStatsDB {
 				+ "into 100 buckets stored as orc tblproperties('transactional'='true')";
 			stmt.executeUpdate(sqlCreateTablePiwikLog);
 
-//            String dropT = "TRUNCATE TABLE "
-//                    + ConnectDB.getUsageStatsDBSchema()
-//                    + ".piwiklog ";
-//            stmt.executeUpdate(dropT);
-//                        logger.info("truncated piwiklog");
-
 			/////////////////////////////////////////
 			// Rule for duplicate inserts @ piwiklog
 			/////////////////////////////////////////

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/UsageStatsExporter.java

@@ -2,7 +2,6 @@
 package eu.dnetlib.oa.graph.usagerawdata.export;
 
 import java.io.IOException;
-import java.sql.SQLException;
 import java.sql.Statement;
 
 import org.apache.hadoop.conf.Configuration;