fixed step1 workflow

This commit is contained in:
Sandro La Bruzzo 2020-03-27 17:07:34 +01:00
parent 1a7a866861
commit 62cc257e5c
3 changed files with 85 additions and 8 deletions

View File

@ -10,6 +10,37 @@
<artifactId>dhp-dedup-scholexplorer</artifactId> <artifactId>dhp-dedup-scholexplorer</artifactId>
<build>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>4.0.1</version>
<executions>
<execution>
<id>scala-compile-first</id>
<phase>initialize</phase>
<goals>
<goal>add-source</goal>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>scala-test-compile</id>
<phase>process-test-resources</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
</configuration>
</plugin>
</plugins>
</build>
<dependencies> <dependencies>
<dependency> <dependency>

View File

@ -0,0 +1,3 @@
# dnet-graph-mapper
Dnet-graph-mapper is a DNET module responsible
of importing the first version of graph into Hadoop Cluster.

View File

@ -1,11 +1,20 @@
<workflow-app name="Create Raw Graph Step 1: import Entities from aggregator to HDFS" xmlns="uri:oozie:workflow:0.5"> <workflow-app name="Create Raw Graph Step 1: import Entities from aggregator to HDFS" xmlns="uri:oozie:workflow:0.5">
<parameters> <parameters>
<property>
<name>reuseContent</name>
<value>false</value>
<description>should import content from the aggregator or reuse a previous version</description>
</property>
<property> <property>
<name>workingPath</name> <name>workingPath</name>
<description>the working dir base path</description> <description>the working dir base path</description>
</property> </property>
<property> <property>
<name>targetPath</name> <name>targetXMLPath</name>
<description>the graph Raw base path</description>
</property>
<property>
<name>targetEntityPath</name>
<description>the graph Raw base path</description> <description>the graph Raw base path</description>
</property> </property>
<property> <property>
@ -29,12 +38,20 @@
<description>mongo database</description> <description>mongo database</description>
</property> </property>
<property> <property>
<name>user</name> <name>entity</name>
<description>HDFS user</description> <description>the entity type</description>
</property> </property>
</parameters> </parameters>
<start to="ResetWorkingPath"/> <start to="ReuseContent"/>
<decision name="ReuseContent">
<switch>
<case to="ResetWorkingPath">${wf:conf('reuseContent') eq false}</case>
<case to="ResetTargetPath">${wf:conf('reuseContent') eq true}</case>
<default to="ResetWorkingPath"/>
</switch>
</decision>
<kill name="Kill"> <kill name="Kill">
@ -43,8 +60,8 @@
<action name="ResetWorkingPath"> <action name="ResetWorkingPath">
<fs> <fs>
<delete path='${targetPath}'/> <delete path='${workingPath}'/>
<mkdir path='${workingPath}'/> <mkdir path='${workingPath}/input'/>
</fs> </fs>
<ok to="ImportEntitiesFromMongo"/> <ok to="ImportEntitiesFromMongo"/>
<error to="Kill"/> <error to="Kill"/>
@ -56,9 +73,8 @@
<job-tracker>${jobTracker}</job-tracker> <job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node> <name-node>${nameNode}</name-node>
<main-class>eu.dnetlib.dhp.sx.graph.ImportDataFromMongo</main-class> <main-class>eu.dnetlib.dhp.sx.graph.ImportDataFromMongo</main-class>
<arg>-t</arg><arg>${targetPath}</arg> <arg>-t</arg><arg>${targetXMLPath}</arg>
<arg>-n</arg><arg>${nameNode}</arg> <arg>-n</arg><arg>${nameNode}</arg>
<arg>-u</arg><arg>${user}</arg>
<arg>-h</arg><arg>${dbhost}</arg> <arg>-h</arg><arg>${dbhost}</arg>
<arg>-p</arg><arg>27017</arg> <arg>-p</arg><arg>27017</arg>
<arg>-dn</arg><arg>${dbName}</arg> <arg>-dn</arg><arg>${dbName}</arg>
@ -66,6 +82,33 @@
<arg>-l</arg><arg>${layout}</arg> <arg>-l</arg><arg>${layout}</arg>
<arg>-i</arg><arg>${interpretation}</arg> <arg>-i</arg><arg>${interpretation}</arg>
</java> </java>
<ok to="ResetTargetPath"/>
<error to="Kill"/>
</action>
<action name="ResetTargetPath">
<fs>
<delete path='${targetEntityPath}'/>
</fs>
<ok to="ConvertXML2Entity"/>
<error to="Kill"/>
</action>
<action name="ConvertXML2Entity">
<spark xmlns="uri:oozie:spark-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<master>yarn-cluster</master>
<mode>cluster</mode>
<name>Import ${entity} and related entities</name>
<class>eu.dnetlib.dhp.sx.graph.SparkScholexplorerGraphImporter</class>
<jar>dhp-graph-mapper-${projectVersion}.jar</jar>
<spark-opts>--executor-memory ${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} ${sparkExtraOPT}</spark-opts>
<arg>-mt</arg> <arg>yarn-cluster</arg>
<arg>--sourcePath</arg><arg>${targetXMLPath}</arg>
<arg>--targetPath</arg><arg>${targetEntityPath}</arg>
<arg>--entity</arg><arg>${entity}</arg>
</spark>
<ok to="End"/> <ok to="End"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>