D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 7 Projects 2 Releases 1 Wiki Activity

workflow works in parallel on 2 activity files

This commit is contained in:
Enrico Ottonello 2020-04-24 18:33:37 +02:00
parent 941e94af06
commit a1861b9eaa
2 changed files with 28 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcid/OrcidAuthorsDOIsDataGen.java
View File

@ -25,11 +25,7 @@ public class OrcidAuthorsDOIsDataGen extends OrcidDSManager {
String tarGzUri =
hdfsServerUri.concat(hdfsOrcidDefaultPath).concat(activitiesFileNameTarGz);
Path outputPath =
new Path(
hdfsServerUri
.concat(hdfsOrcidDefaultPath)
.concat(outputAuthorsDOIsPath)
.concat("authors_dois.seq"));
new Path(hdfsServerUri.concat(hdfsOrcidDefaultPath).concat(outputAuthorsDOIsPath));
ActivitiesDecompressor.parseGzActivities(conf, tarGzUri, outputPath);
}

32
dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_authors_dois_data/oozie_app/workflow.xml
View File

@ -18,11 +18,16 @@
<delete path='${workingPath_activities}/output'/>
<mkdir path='${workingPath_activities}/output'/>
</fs>
<ok to="GenerateOrcidAuthorsDOIsData"/>
<ok to="fork_generate_orcid_authors_dois"/>
<error to="Kill"/>
</action>
<action name="GenerateOrcidAuthorsDOIsData">
<fork name = "fork_generate_orcid_authors_dois">
<path start = "Generate_Orcid_Authors_DOIs_0"/>
<path start = "Generate_Orcid_Authors_DOIs_1"/>
</fork>
<action name="Generate_Orcid_Authors_DOIs_0">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
@ -30,10 +35,27 @@
<arg>-d</arg><arg>${workingPath_activities}/</arg>
<arg>-n</arg><arg>${nameNode}</arg>
<arg>-f</arg><arg>ORCID_2019_activites_0.tar.gz</arg>
<arg>-o</arg><arg>output/</arg>
<arg>-o</arg><arg>output/authors_dois_0.seq</arg>
</java>
<ok to="End"/>
<ok to="join_node"/>
<error to="Kill"/>
</action>
<end name="End"/>
<action name="Generate_Orcid_Authors_DOIs_1">
<java>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
<arg>-d</arg><arg>${workingPath_activities}/</arg>
<arg>-n</arg><arg>${nameNode}</arg>
<arg>-f</arg><arg>ORCID_2019_activites_1.tar.gz</arg>
<arg>-o</arg><arg>output/authors_dois_1.seq</arg>
</java>
<ok to="join_node"/>
<error to="Kill"/>
</action>
<join name = "join_node" to = "End"/>
<end name="End"/>
</workflow-app>