changed workflow flow to possibly parallelize also the programme and project preparation steps

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java

@@ -18,7 +18,7 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme;
+
 import eu.dnetlib.dhp.actionmanager.project.utils.CSVProject;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
@@ -32,7 +32,7 @@ public class PrepareProjects {
 
 	private static final Logger log = LoggerFactory.getLogger(PrepareProgramme.class);
 	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-	//private static final HashMap<String, CSVProgramme> programmeMap = new HashMap<>();
+
 
 	public static void main(String[] args) throws Exception {
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/EXCELParser.java

@@ -26,7 +26,6 @@ public class EXCELParser {
 		throws ClassNotFoundException, IOException, IllegalAccessException, InstantiationException,
 		InvalidFormatException {
 
-		// OPCPackage pkg = OPCPackage.open(httpConnector.getInputSourceAsStream(URL));
 		OPCPackage pkg = OPCPackage.open(file);
 		XSSFWorkbook wb = new XSSFWorkbook(pkg);
 
@@ -58,7 +57,6 @@ public class EXCELParser {
 
 				for (int i = 0; i < headers.size(); i++) {
 					Cell cell = row.getCell(i);
-					String value = dataFormatter.formatCellValue(cell);
 					FieldUtils.writeField(cc, headers.get(i), dataFormatter.formatCellValue(cell), true);
 
 				}

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml

@@ -41,11 +41,17 @@
 
 
     <fork name="fork_get_info">
-        <path start="get_project_file"/>
+        <path start="fork_get_projects"/>
         <path start="get_programme_file"/>
         <path start="get_topic_file"/>
+
+    </fork>
+
+    <fork name="fork_get_projects">
+        <path start="get_project_file"/>
         <path start="read_projects"/>
     </fork>
+
     <action name="get_project_file">
         <java>
             <main-class>eu.dnetlib.dhp.actionmanager.project.utils.ReadCSV</main-class>
@@ -54,7 +60,7 @@
             <arg>--hdfsPath</arg><arg>${workingDir}/projects</arg>
             <arg>--classForName</arg><arg>eu.dnetlib.dhp.actionmanager.project.utils.CSVProject</arg>
         </java>
-        <ok to="wait"/>
+        <ok to="wait_projects"/>
         <error to="Kill"/>
     </action>
 
@@ -66,7 +72,7 @@
             <arg>--hdfsPath</arg><arg>${workingDir}/programme</arg>
             <arg>--classForName</arg><arg>eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme</arg>
         </java>
-        <ok to="wait"/>
+        <ok to="prepare_programme"/>
         <error to="Kill"/>
     </action>
 
@@ -92,12 +98,10 @@
             <arg>--postgresUser</arg><arg>${postgresUser}</arg>
             <arg>--postgresPassword</arg><arg>${postgresPassword}</arg>
         </java>
-        <ok to="wait"/>
+        <ok to="wait_projects"/>
         <error to="Kill"/>
     </action>
 
-    <join name="wait" to="prepare_programme"/>
-
     <action name="prepare_programme">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
@@ -118,10 +122,15 @@
             <arg>--programmePath</arg><arg>${workingDir}/programme</arg>
             <arg>--outputPath</arg><arg>${workingDir}/preparedProgramme</arg>
         </spark>
-        <ok to="prepare_project"/>
+        <ok to="wait"/>
         <error to="Kill"/>
     </action>
 
+    <join name="wait" to="create_updates"/>
+
+    <join name="wait_projects" to="prepare_project"/>
+
+
     <action name="prepare_project">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
@@ -143,7 +152,7 @@
             <arg>--outputPath</arg><arg>${workingDir}/preparedProjects</arg>
             <arg>--dbProjectPath</arg><arg>${workingDir}/dbProjects</arg>
         </spark>
-        <ok to="create_updates"/>
+        <ok to="wait"/>
         <error to="Kill"/>
     </action>