forked from D-Net/dnet-hadoop
removed 'stored as parquet' from views..
This commit is contained in:
parent
915f758c82
commit
ddd34087c2
|
|
@ -0,0 +1,57 @@
|
|||
export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
|
||||
export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
|
||||
if ! [ -L $link_folder ]
|
||||
then
|
||||
rm -Rf "$link_folder"
|
||||
ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
|
||||
fi
|
||||
|
||||
function copydb() {
|
||||
db=$1
|
||||
|
||||
# copy the databases from ocean to impala
|
||||
|
||||
#echo "copying $db"
|
||||
hadoop distcp -Dmapreduce.map.memory.mb=6144 -pb hdfs://nameservice1/user/hive/warehouse/${db}.db hdfs://impala-cluster-mn2.openaire.eu:8020/tmp
|
||||
|
||||
# change ownership to impala
|
||||
hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/${db}.db
|
||||
|
||||
# create the databases
|
||||
impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade";
|
||||
impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}";
|
||||
|
||||
echo "creating schema for ${db}"
|
||||
for i in `impala-shell -d ${db} --delimited -q "show tables"`;
|
||||
do
|
||||
impala-shell -d ${db} --delimited -q "show create table $i";
|
||||
done | sed 's/"$/;/' | sed 's/^"//' | impala-shell -i impala-cluster-dn1.openaire.eu -c -f -
|
||||
|
||||
# run the same command twice because we may have failures in the first run (due to views pointing to the same db)
|
||||
for i in `impala-shell -d ${db} --delimited -q "show tables"`;
|
||||
do
|
||||
impala-shell -d ${db} --delimited -q "show create table $i";
|
||||
done | sed 's/"$/;/' | sed 's/^"//' | impala-shell -i impala-cluster-dn1.openaire.eu -c -f -
|
||||
|
||||
# load the data from /tmp in the respective tables
|
||||
echo "copying data in tables and computing stats"
|
||||
for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`;
|
||||
do
|
||||
impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "load data inpath '/tmp/${db}.db/$i' into table $i";
|
||||
impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "compute stats $i";
|
||||
done
|
||||
|
||||
# deleting the remaining directory from hdfs
|
||||
hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -rm -R /tmp/${db}.db
|
||||
}
|
||||
|
||||
STATS_DB=$1
|
||||
MONITOR_DB=$2
|
||||
OBSERVATORY_DB=$3
|
||||
EXT_DB=$4
|
||||
|
||||
copydb $EXT_DB
|
||||
copydb $STATS_DB
|
||||
copydb $MONITOR_DB
|
||||
copydb $OBSERVATORY_DB
|
||||
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
|
||||
export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
|
||||
if ! [ -L $link_folder ]
|
||||
then
|
||||
rm -Rf "$link_folder"
|
||||
ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
|
||||
fi
|
||||
|
||||
function createShadowDB() {
|
||||
SOURCE=$1
|
||||
SHADOW=$2
|
||||
|
||||
impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${SHADOW}";
|
||||
impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "show tables" | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f -
|
||||
impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE} -q "show tables" | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f -
|
||||
}
|
||||
|
||||
STATS_DB=$1
|
||||
STATS_DB_SHADOW=$2
|
||||
MONITOR_DB=$3
|
||||
MONITOR_DB_SHADOW=$4
|
||||
OBSERVATORY_DB=$5
|
||||
OBSERVATORY_DB_SHADOW=$6
|
||||
|
||||
createShadowDB $STATS_DB $STATS_DB_SHADOW
|
||||
createShadowDB $MONITOR_DB $MONITOR_DB_SHADOW
|
||||
createShadowDB $OBSERVATORY_DB $OBSERVATORY_DB_SHADOW
|
||||
|
|
@ -102,7 +102,7 @@ WHERE otherresearchproduct_tmp.id IN (SELECT distinct r.id
|
|||
AND pr.id = p.id
|
||||
AND to_date(r.date) - to_date(p.enddate) > 0);
|
||||
|
||||
CREATE OR REPLACE VIEW ${stats_db_name}.project_results_publication STORED AS PARQUET AS
|
||||
CREATE OR REPLACE VIEW ${stats_db_name}.project_results_publication AS
|
||||
SELECT result_projects.id AS result,
|
||||
result_projects.project AS project_results,
|
||||
result.date as resultdate,
|
||||
|
|
@ -113,7 +113,4 @@ FROM ${stats_db_name}.result_projects,
|
|||
${stats_db_name}.project
|
||||
WHERE result_projects.id = result.id
|
||||
AND result.type = 'publication'
|
||||
AND project.id = result_projects.project;
|
||||
|
||||
-- ANALYZE TABLE ${stats_db_name}.project COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project COMPUTE STATISTICS FOR COLUMNS;
|
||||
AND project.id = result_projects.project;
|
||||
|
|
@ -25,7 +25,7 @@ SELECT substr(p.id, 4) as id, licenses.value as type
|
|||
from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.license) instances as licenses
|
||||
where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_licenses STORED AS PARQUET AS
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_licenses AS
|
||||
SELECT * FROM ${stats_db_name}.publication_licenses
|
||||
UNION ALL
|
||||
SELECT * FROM ${stats_db_name}.dataset_licenses
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ select substr(r.id, 4) as id, inst.refereed.classname as refereed
|
|||
from ${openaire_db_name}.otherresearchproduct r lateral view explode(r.instance) instances as inst
|
||||
where r.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_refereed STORED AS PARQUET as
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_refereed as
|
||||
select * from ${stats_db_name}.publication_refereed
|
||||
union all
|
||||
select * from ${stats_db_name}.dataset_refereed
|
||||
|
|
|
|||
Loading…
Reference in New Issue