removed 'stored as parquet' from views..

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh

@@ -0,0 +1,57 @@
+export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
+export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
+if ! [ -L $link_folder ]
+then
+    rm -Rf "$link_folder"
+    ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
+fi
+
+function copydb() {
+  db=$1
+
+  # copy the databases from ocean to impala
+
+  #echo "copying $db"
+  hadoop distcp -Dmapreduce.map.memory.mb=6144 -pb hdfs://nameservice1/user/hive/warehouse/${db}.db hdfs://impala-cluster-mn2.openaire.eu:8020/tmp
+
+  # change ownership to impala
+  hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/${db}.db
+
+  # create the databases
+  impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade";
+  impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}";
+
+  echo "creating schema for ${db}"
+  for i in `impala-shell -d ${db} --delimited  -q "show tables"`;
+    do
+      impala-shell -d ${db} --delimited  -q "show create table $i";
+    done |  sed 's/"$/;/' | sed 's/^"//' | impala-shell -i impala-cluster-dn1.openaire.eu -c -f -
+
+  # run the same command twice because we may have failures in the first run (due to views pointing to the same db)
+  for i in `impala-shell -d ${db} --delimited  -q "show tables"`;
+    do
+      impala-shell -d ${db} --delimited  -q "show create table $i";
+    done |  sed 's/"$/;/' | sed 's/^"//' | impala-shell -i impala-cluster-dn1.openaire.eu -c -f -
+
+  # load the data from /tmp in the respective tables
+  echo "copying data in tables and computing stats"
+  for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited  -q "show tables"`;
+      do
+        impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "load data inpath '/tmp/${db}.db/$i' into table $i";
+        impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "compute stats $i";
+      done
+
+  # deleting the remaining directory from hdfs
+  hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -rm -R /tmp/${db}.db
+}
+
+STATS_DB=$1
+MONITOR_DB=$2
+OBSERVATORY_DB=$3
+EXT_DB=$4
+
+copydb $EXT_DB
+copydb $STATS_DB
+copydb $MONITOR_DB
+copydb $OBSERVATORY_DB
+

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/finalizeImpalaCluster.sh

@@ -0,0 +1,27 @@
+export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
+export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
+if ! [ -L $link_folder ]
+then
+    rm -Rf "$link_folder"
+    ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
+fi
+
+function createShadowDB() {
+  SOURCE=$1
+  SHADOW=$2
+
+  impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${SHADOW}";
+  impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "show tables" | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f -
+  impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE} -q "show tables" | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f -
+}
+
+STATS_DB=$1
+STATS_DB_SHADOW=$2
+MONITOR_DB=$3
+MONITOR_DB_SHADOW=$4
+OBSERVATORY_DB=$5
+OBSERVATORY_DB_SHADOW=$6
+
+createShadowDB $STATS_DB $STATS_DB_SHADOW
+createShadowDB $MONITOR_DB $MONITOR_DB_SHADOW
+createShadowDB $OBSERVATORY_DB $OBSERVATORY_DB_SHADOW

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step11.sql

@@ -102,7 +102,7 @@ WHERE otherresearchproduct_tmp.id IN (SELECT distinct r.id
                                         AND pr.id = p.id
                                         AND to_date(r.date) - to_date(p.enddate) > 0);
 
-CREATE OR REPLACE VIEW ${stats_db_name}.project_results_publication STORED AS PARQUET AS
+CREATE OR REPLACE VIEW ${stats_db_name}.project_results_publication AS
 SELECT result_projects.id          AS result,
        result_projects.project     AS project_results,
        result.date                 as resultdate,
@@ -113,7 +113,4 @@ FROM ${stats_db_name}.result_projects,
      ${stats_db_name}.project
 WHERE result_projects.id = result.id
   AND result.type = 'publication'
-  AND project.id = result_projects.project;
+  AND project.id = result_projects.project;
-
--- ANALYZE TABLE ${stats_db_name}.project COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.project COMPUTE STATISTICS FOR COLUMNS;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql

@@ -25,7 +25,7 @@ SELECT substr(p.id, 4) as id, licenses.value as type
 from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.license) instances as licenses
 where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
 
-CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_licenses STORED AS PARQUET AS
+CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_licenses AS
 SELECT * FROM ${stats_db_name}.publication_licenses
 UNION ALL
 SELECT * FROM ${stats_db_name}.dataset_licenses

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql

@@ -26,7 +26,7 @@ select substr(r.id, 4) as id, inst.refereed.classname as refereed
 from ${openaire_db_name}.otherresearchproduct r lateral view explode(r.instance) instances as inst
 where r.datainfo.deletedbyinference=false;
 
-CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_refereed STORED AS PARQUET as
+CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_refereed as
 select * from ${stats_db_name}.publication_refereed
 union all
 select * from ${stats_db_name}.dataset_refereed