antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

added new hive opts

This commit is contained in:
Antonis Lempesis 2022-04-29 12:49:27 +03:00
parent 5772f92dba
commit 0353f93d54
2 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/indicators.sh
View File

@ -8,12 +8,13 @@ fi
export TARGET=$1 export TARGET=$1
export SCRIPT_PATH=$2 export SCRIPT_PATH=$2
export HIVE_OPTS="-hiveconf mapred.job.queue.name=analytics -hiveconf hive.spark.client.connect.timeout=120000ms -hiveconf hive.spark.client.server.connect.timeout=300000ms"
echo "Getting file from " $SCRIPT_PATH echo "Getting file from " $SCRIPT_PATH
hdfs dfs -copyToLocal $SCRIPT_PATH hdfs dfs -copyToLocal $SCRIPT_PATH
echo "Creating indicators" echo "Creating indicators"
hive --database ${TARGET} -e "show tables" | grep -v WARN | sed "s/^\(.*\)/analyze table ${TARGET}.\1 compute statistics;/" > foo hive $HIVE_OPTS --database ${TARGET} -e "show tables" | grep -v WARN | sed "s/^\(.*\)/analyze table ${TARGET}.\1 compute statistics;/" > foo
hive -f foo hive $HIVE_OPTS -f foo
hive --database ${TARGET} -f step16-createIndicatorsTables.sql hive $HIVE_OPTS --database ${TARGET} -f step16-createIndicatorsTables.sql
echo "Indicators created" echo "Indicators created"

10
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -67,10 +67,14 @@
<name>hive.txn.timeout</name> <name>hive.txn.timeout</name>
<value>${hive_timeout}</value> <value>${hive_timeout}</value>
</property> </property>
<property>
<name>mapred.job.queue.name</name>
<value>analytics</value>
</property>
</configuration> </configuration>
</global> </global>
<start to="Step1"/> <start to="Step16-createIndicatorsTables"/>
<kill name="Kill"> <kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
@ -276,7 +280,7 @@
<argument>${wf:appPath()}/scripts/step16-createIndicatorsTables.sql</argument> <argument>${wf:appPath()}/scripts/step16-createIndicatorsTables.sql</argument>
<file>indicators.sh</file> <file>indicators.sh</file>
</shell> </shell>
<ok to="Step16_1-definitions"/> <ok to="End"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>
@ -414,4 +418,4 @@
</action> </action>
<end name="End"/> <end name="End"/>
</workflow-app> </workflow-app>