1
0
Fork 0

added config for new propagation

This commit is contained in:
Miriam Baglioni 2020-02-18 17:30:44 +01:00
parent b736a9581c
commit b81e6af429
2 changed files with 4 additions and 26 deletions

View File

@ -10,17 +10,5 @@
"paramLongName":"sourcePath",
"paramDescription": "the path of the sequencial file to read",
"paramRequired": true
},
{
"paramName":"wl",
"paramLongName":"whitelist",
"paramDescription": "datasource id that will be considered even if not in the allowed typology list. Split by ;",
"paramRequired": true
},
{
"paramName":"at",
"paramLongName":"allowedtypes",
"paramDescription": "the types of the allowed datasources. Split by ;",
"paramRequired": true
}
]

View File

@ -1,17 +1,9 @@
<workflow-app name="country_propagation" xmlns="uri:oozie:workflow:0.5">
<workflow-app name="affiliation_from_instrepo_propagation" xmlns="uri:oozie:workflow:0.5">
<parameters>
<property>
<name>sourcePath</name>
<description>the source path</description>
</property>
<property>
<name>whitelist</name>
<description>the white list</description>
</property>
<property>
<name>allowedtypes</name>
<description>the allowed types</description>
</property>
<property>
<name>sparkDriverMemory</name>
<description>memory for driver process</description>
@ -26,20 +18,20 @@
</property>
</parameters>
<start to="CountryPropagation"/>
<start to="AffiliationPropagation"/>
<kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
</kill>
<action name="CountryPropagation">
<action name="AffiliationPropagation">
<spark xmlns="uri:oozie:spark-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<master>yarn-cluster</master>
<mode>cluster</mode>
<name>CountryPropagation</name>
<class>eu.dnetlib.dhp.countrypropagation.SparkCountryPropagationJob</class>
<class>eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob</class>
<jar>dhp-propagation-${projectVersion}.jar</jar>
<spark-opts>--executor-memory ${sparkExecutorMemory}
--executor-cores ${sparkExecutorCores}
@ -49,8 +41,6 @@
</spark-opts>
<arg>-mt</arg> <arg>yarn-cluster</arg>
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
<arg>--whitelist</arg><arg>${whitelist}</arg>
<arg>--allowedtypes</arg><arg>${allowedtypes}</arg>
</spark>
<ok to="End"/>
<error to="Kill"/>