added new hive opts

This commit is contained in:
Antonis Lempesis 2022-04-29 12:49:27 +03:00
parent 5772f92dba
commit 0353f93d54
2 changed files with 12 additions and 7 deletions

View File

@ -8,12 +8,13 @@ fi
export TARGET=$1
export SCRIPT_PATH=$2
export HIVE_OPTS="-hiveconf mapred.job.queue.name=analytics -hiveconf hive.spark.client.connect.timeout=120000ms -hiveconf hive.spark.client.server.connect.timeout=300000ms"
echo "Getting file from " $SCRIPT_PATH
hdfs dfs -copyToLocal $SCRIPT_PATH
echo "Creating indicators"
hive --database ${TARGET} -e "show tables" | grep -v WARN | sed "s/^\(.*\)/analyze table ${TARGET}.\1 compute statistics;/" > foo
hive -f foo
hive --database ${TARGET} -f step16-createIndicatorsTables.sql
echo "Indicators created"
hive $HIVE_OPTS --database ${TARGET} -e "show tables" | grep -v WARN | sed "s/^\(.*\)/analyze table ${TARGET}.\1 compute statistics;/" > foo
hive $HIVE_OPTS -f foo
hive $HIVE_OPTS --database ${TARGET} -f step16-createIndicatorsTables.sql
echo "Indicators created"

View File

@ -67,10 +67,14 @@
<name>hive.txn.timeout</name>
<value>${hive_timeout}</value>
</property>
<property>
<name>mapred.job.queue.name</name>
<value>analytics</value>
</property>
</configuration>
</global>
<start to="Step1"/>
<start to="Step16-createIndicatorsTables"/>
<kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
@ -276,7 +280,7 @@
<argument>${wf:appPath()}/scripts/step16-createIndicatorsTables.sql</argument>
<file>indicators.sh</file>
</shell>
<ok to="Step16_1-definitions"/>
<ok to="End"/>
<error to="Kill"/>
</action>
@ -414,4 +418,4 @@
</action>
<end name="End"/>
</workflow-app>
</workflow-app>