lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

Use the "nameservice1" virtual name which should resolve automatically to the active node in the Ocean Cluster.

This commit is contained in:
Lampros Smyrnaios 2024-02-05 16:38:27 +02:00
parent ad99940c3b
commit 4fd242155e
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh
View File

@ -10,9 +10,8 @@ export PROD_USAGE_STATS_DB="openaire_prod_usage_stats"
# Set the active HDFS node of OCEAN and IMPALA cluster.
OCEAN_DOMAIN='iis-cdh5-test-m1.ocean.icm.edu.pl'
OCEAN_HDFS_NODE='hdfs://'${OCEAN_DOMAIN}':8020'
echo "Active OCEAN HDFS Node: ${OCEAN_HDFS_NODE}"
OCEAN_HDFS_NODE='hdfs://nameservice1'
echo "OCEAN HDFS virtual-name which resolves automatically to the active-node: ${OCEAN_HDFS_NODE}"
IMPALA_HDFS_NODE=''
if hdfs dfs -test -e hdfs://impala-cluster-mn1.openaire.eu >/dev/null 2>&1; then
@ -71,7 +70,7 @@ function copydb() {
# Since there are many DBs handled in this script and many more may be added, we cannot easily keep track of the views of all those tables, so leave this extra loop for now.
for (( k = 1; k <= 2; k++ )); do
echo -e "Create tables iteration_${k}"
echo -e "\nCreate tables iteration_${k}\n"
for i in `hive -e "use $db; show tables;" | sed 's/WARN:.*//g'`; do
CURRENT_PRQ_FILE=`hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -ls -C "/tmp/${TEMP_SUBDIR}/${db}.db/${i}/*.parq" | head -1`
if [[ -z "$CURRENT_PRQ_FILE" ]]; then