157 lines
5.9 KiB
XML
157 lines
5.9 KiB
XML
<workflow-app name="Native records migration" xmlns="uri:oozie:workflow:0.5">
|
|
<parameters>
|
|
<property>
|
|
<name>datasourceId</name>
|
|
<description>datasource identifier</description>
|
|
</property>
|
|
<property>
|
|
<name>datasourceName</name>
|
|
<description>datasource name</description>
|
|
</property>
|
|
<property>
|
|
<name>nsPrefix</name>
|
|
<description>datasource namespace prefix</description>
|
|
</property>
|
|
|
|
<property>
|
|
<name>metadataEncoding</name>
|
|
<description> The type of the metadata XML/JSON</description>
|
|
</property>
|
|
<property>
|
|
<name>workflowId</name>
|
|
<description>The identifier of the workflow</description>
|
|
</property>
|
|
<property>
|
|
<name>targetMdStoreID</name>
|
|
<description>The identifier of the target mdStore</description>
|
|
</property>
|
|
<property>
|
|
<name>sourceMDStoreId</name>
|
|
<description>The identifier of the source mdStore</description>
|
|
</property>
|
|
|
|
<property>
|
|
<name>mdStoreManagerURI</name>
|
|
<description>The URI of the MDStore Manager</description>
|
|
</property>
|
|
|
|
<property>
|
|
<name>dnetMessageManagerURL</name>
|
|
<description>The URI of the Dnet Message Manager</description>
|
|
</property>
|
|
|
|
<property>
|
|
<name>mongoDbUri</name>
|
|
<description>MongoDB server URI</description>
|
|
</property>
|
|
|
|
|
|
<property>
|
|
<name>mongoDbName</name>
|
|
<value>mdstore</value>
|
|
<description>MDStore database name</description>
|
|
</property>
|
|
|
|
<property>
|
|
<name>collection_java_xmx</name>
|
|
<value>-Xmx200m</value>
|
|
<description>Used to configure the heap size for the map JVM process. Should be 80% of mapreduce.map.memory.mb.</description>
|
|
</property>
|
|
|
|
|
|
</parameters>
|
|
|
|
<global>
|
|
<job-tracker>${jobTracker}</job-tracker>
|
|
<name-node>${nameNode}</name-node>
|
|
</global>
|
|
|
|
<start to="StartTransaction"/>
|
|
|
|
<kill name="Kill">
|
|
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
|
</kill>
|
|
|
|
<action name="StartTransaction">
|
|
<java>
|
|
<main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
|
|
<java-opts>${collection_java_xmx}</java-opts>
|
|
<arg>--action</arg><arg>NEW_VERSION</arg>
|
|
<arg>--mdStoreID</arg><arg>${targetMdStoreID}</arg>
|
|
<arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
|
|
<capture-output/>
|
|
</java>
|
|
<ok to="CopyNativeRecords"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<action name="CopyNativeRecords">
|
|
<java>
|
|
<main-class>eu.dnetlib.dhp.migration.MongoDbMDStoreCollectionReader</main-class>
|
|
<java-opts>${collection_java_xmx}</java-opts>
|
|
<arg>--mongoDbUri</arg><arg>${mongoDbUri}</arg>
|
|
<arg>--mongoDbName</arg><arg>${mongoDbName}</arg>
|
|
<arg>--sourceMDStoreId</arg><arg>${sourceMDStoreId}</arg>
|
|
<arg>--namenode</arg><arg>${nameNode}</arg>
|
|
<arg>--workflowId</arg><arg>${workflowId}</arg>
|
|
<arg>--dnetMessageManagerURL</arg><arg>${dnetMessageManagerURL}</arg>
|
|
<arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
|
|
</java>
|
|
<ok to="MigrateNativeStoreSparkJob"/>
|
|
<error to="RollBack"/>
|
|
</action>
|
|
|
|
<action name="MigrateNativeStoreSparkJob">
|
|
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
<master>yarn</master>
|
|
<mode>cluster</mode>
|
|
<name>Migrate Native MetadataStore</name>
|
|
<class>eu.dnetlib.dhp.migration.MigrateNativeStoreSparkJob</class>
|
|
<jar>dhp-aggregation-${projectVersion}.jar</jar>
|
|
<spark-opts>
|
|
--executor-memory=${sparkExecutorMemory}
|
|
--executor-cores=${sparkExecutorCores}
|
|
--driver-memory=${sparkDriverMemory}
|
|
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
</spark-opts>
|
|
<arg>--encoding</arg><arg>${metadataEncoding}</arg>
|
|
<arg>--datasourceId</arg><arg>${datasourceId}</arg>
|
|
<arg>--datasourceName</arg><arg>${datasourceName}</arg>
|
|
<arg>--nsPrefix</arg><arg>${nsPrefix}</arg>
|
|
<arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
|
|
</spark>
|
|
<ok to="CommitVersion"/>
|
|
<error to="RollBack"/>
|
|
</action>
|
|
|
|
<action name="CommitVersion">
|
|
<java>
|
|
<main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
|
|
<java-opts>${collection_java_xmx}</java-opts>
|
|
<arg>--action</arg><arg>COMMIT</arg>
|
|
<arg>--namenode</arg><arg>${nameNode}</arg>
|
|
<arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
|
|
<arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
|
|
</java>
|
|
<ok to="End"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<action name="RollBack">
|
|
<java>
|
|
<main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
|
|
<java-opts>${collection_java_xmx}</java-opts>
|
|
<arg>--action</arg><arg>ROLLBACK</arg>
|
|
<arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
|
|
<arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
|
|
</java>
|
|
<ok to="Kill"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<end name="End"/>
|
|
|
|
</workflow-app> |