pispis
/
dnet-hadoop
forked from antonis.lempesis/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

deleted folders with merged data immediately before merge phases

This commit is contained in:
Enrico Ottonello 2021-04-28 11:25:25 +02:00
parent 2129e9caa7
commit c537986b7c
1 changed files with 18 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_updates_download/oozie_app/workflow.xml
View File

@ -31,7 +31,7 @@
</property> </property>
<property> <property>
<name>sparkExecutorMemory</name> <name>sparkExecutorMemory</name>
<value>2G</value> <value>4G</value>
<description>memory for individual executor</description> <description>memory for individual executor</description>
</property> </property>
<property> <property>
@ -190,6 +190,14 @@
<arg>-o</arg><arg>downloads/updated_works</arg> <arg>-o</arg><arg>downloads/updated_works</arg>
<arg>-t</arg><arg>${token}</arg> <arg>-t</arg><arg>${token}</arg>
</spark> </spark>
<ok to="ResetNewAuthors"/>
<error to="Kill"/>
</action>
<action name="ResetNewAuthors">
<fs>
<delete path='${workingPath}/orcid_dataset/new_authors'/>
</fs>
<ok to="UpdateOrcidAuthors"/> <ok to="UpdateOrcidAuthors"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>
@ -217,6 +225,14 @@
<arg>-o</arg><arg>-</arg> <arg>-o</arg><arg>-</arg>
<arg>-t</arg><arg>-</arg> <arg>-t</arg><arg>-</arg>
</spark> </spark>
<ok to="ResetNewWorks"/>
<error to="Kill"/>
</action>
<action name="ResetNewWorks">
<fs>
<delete path='${workingPath}/orcid_dataset/new_works'/>
</fs>
<ok to="UpdateOrcidWorks"/> <ok to="UpdateOrcidWorks"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>
@ -270,15 +286,6 @@
<arg>${workingPath}/orcid_dataset/new_works/*</arg> <arg>${workingPath}/orcid_dataset/new_works/*</arg>
<arg>${workingPath}/orcid_dataset/works</arg> <arg>${workingPath}/orcid_dataset/works</arg>
</distcp> </distcp>
<ok to="CleanWorkingPath"/>
<error to="Kill"/>
</action>
<action name="CleanWorkingPath">
<fs>
<delete path='${workingPath}/orcid_dataset/new_authors'/>
<delete path='${workingPath}/orcid_dataset/new_works'/>
</fs>
<ok to="updateLastOrcidAuthorsDataset"/> <ok to="updateLastOrcidAuthorsDataset"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>