Merge branch '8172_impact_indicators_workflow' of https://code-repo.d4science.org/D-Net/dnet-hadoop into 8172_impact_indicators_workflow

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java

@@ -9,7 +9,6 @@ import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipProjectModel;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
@@ -25,8 +24,9 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipResultModel;
 import eu.dnetlib.dhp.actionmanager.bipmodel.BipScore;
+import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipProjectModel;
+import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipResultModel;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
 import eu.dnetlib.dhp.schema.action.AtomicAction;
@@ -89,8 +89,7 @@ public class SparkAtomicActionScoreJob implements Serializable {
 					default:
 						throw new RuntimeException("Unknown target entity: " + targetEntity);
 				}
-			}
+			});
-		);
 	}
 
 	private static <I extends Project> void prepareProjects(SparkSession spark, String inputPath, String outputPath) {
@@ -98,17 +97,18 @@ public class SparkAtomicActionScoreJob implements Serializable {
 		// read input bip project scores
 		Dataset<BipProjectModel> projectScores = readPath(spark, inputPath, BipProjectModel.class);
 
-		projectScores.map( (MapFunction<BipProjectModel, Project>) bipProjectScores -> {
+		projectScores.map((MapFunction<BipProjectModel, Project>) bipProjectScores -> {
 			Project project = new Project();
 			project.setId(bipProjectScores.getProjectId());
 			project.setMeasures(bipProjectScores.toMeasures());
 			return project;
 		}, Encoders.bean(Project.class))
-		.toJavaRDD()
+			.toJavaRDD()
-		.map(p -> new AtomicAction(Project.class, p))
+			.map(p -> new AtomicAction(Project.class, p))
-		.mapToPair( aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
+			.mapToPair(
-				new Text(OBJECT_MAPPER.writeValueAsString(aa))))
+				aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
-		.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class);
+					new Text(OBJECT_MAPPER.writeValueAsString(aa))))
+			.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class);
 
 	}
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipmodel/score/deserializers/BipProjectModel.java

@@ -1,69 +1,74 @@
+
 package eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers;
 
-import com.opencsv.bean.CsvBindByPosition;
+import static eu.dnetlib.dhp.actionmanager.Constants.*;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
-import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import eu.dnetlib.dhp.schema.oaf.Measure;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import static eu.dnetlib.dhp.actionmanager.Constants.*;
+import com.opencsv.bean.CsvBindByPosition;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Measure;
+import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
 
 @NoArgsConstructor
 @AllArgsConstructor
 @Getter
 @Setter
 public class BipProjectModel {
-    String projectId;
+	String projectId;
 
-    String numOfInfluentialResults;
+	String numOfInfluentialResults;
 
-    String numOfPopularResults;
+	String numOfPopularResults;
 
-    String totalImpulse;
+	String totalImpulse;
 
-    String totalCitationCount;
+	String totalCitationCount;
 
-    // each project bip measure has exactly one value, hence one key-value pair
+	// each project bip measure has exactly one value, hence one key-value pair
-    private Measure createMeasure(String measureId, String measureValue) {
+	private Measure createMeasure(String measureId, String measureValue) {
 
-        KeyValue kv = new KeyValue();
+		KeyValue kv = new KeyValue();
-        kv.setKey("score");
+		kv.setKey("score");
-        kv.setValue(measureValue);
+		kv.setValue(measureValue);
-        kv.setDataInfo(
+		kv
-            OafMapperUtils.dataInfo(
+			.setDataInfo(
-                false,
+				OafMapperUtils
-                UPDATE_DATA_INFO_TYPE,
+					.dataInfo(
-                true,
+						false,
-                false,
+						UPDATE_DATA_INFO_TYPE,
-                OafMapperUtils.qualifier(
+						true,
-                    UPDATE_MEASURE_BIP_CLASS_ID,
+						false,
-                    UPDATE_CLASS_NAME,
+						OafMapperUtils
-                    ModelConstants.DNET_PROVENANCE_ACTIONS,
+							.qualifier(
-                    ModelConstants.DNET_PROVENANCE_ACTIONS),
+								UPDATE_MEASURE_BIP_CLASS_ID,
-    "")
+								UPDATE_CLASS_NAME,
-        );
+								ModelConstants.DNET_PROVENANCE_ACTIONS,
+								ModelConstants.DNET_PROVENANCE_ACTIONS),
+						""));
 
-        Measure measure = new Measure();
+		Measure measure = new Measure();
-        measure.setId(measureId);
+		measure.setId(measureId);
-        measure.setUnit(Collections.singletonList(kv));
+		measure.setUnit(Collections.singletonList(kv));
-        return measure;
+		return measure;
-    }
+	}
-    public List<Measure> toMeasures() {
+
-        return Arrays.asList(
+	public List<Measure> toMeasures() {
-            createMeasure("numOfInfluentialResults", numOfInfluentialResults),
+		return Arrays
-            createMeasure("numOfPopularResults", numOfPopularResults),
+			.asList(
-            createMeasure("totalImpulse", totalImpulse),
+				createMeasure("numOfInfluentialResults", numOfInfluentialResults),
-            createMeasure("totalCitationCount", totalCitationCount)
+				createMeasure("numOfPopularResults", numOfPopularResults),
-        );
+				createMeasure("totalImpulse", totalImpulse),
-    }
+				createMeasure("totalCitationCount", totalCitationCount));
+	}
 
 }

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipmodel/score/deserializers/BipResultModel.java

@@ -1,13 +1,13 @@
 
 package eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers;
 
-import eu.dnetlib.dhp.actionmanager.bipmodel.Score;
-
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 
+import eu.dnetlib.dhp.actionmanager.bipmodel.Score;
+
 /**
  * Class that maps the model of the bipFinder! input data.
  * Only needed for deserialization purposes

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java

@@ -24,8 +24,8 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipResultModel;
 import eu.dnetlib.dhp.actionmanager.bipmodel.BipScore;
+import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipResultModel;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
 import eu.dnetlib.dhp.schema.common.ModelConstants;

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJobTest.java

@@ -7,8 +7,6 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
-import eu.dnetlib.dhp.schema.oaf.Project;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.spark.SparkConf;
@@ -28,6 +26,8 @@ import org.slf4j.LoggerFactory;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import eu.dnetlib.dhp.schema.action.AtomicAction;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Project;
 import eu.dnetlib.dhp.schema.oaf.Result;
 
 public class SparkAtomicActionScoreJobTest {
@@ -73,15 +73,16 @@ public class SparkAtomicActionScoreJobTest {
 	}
 
 	private void runJob(String inputPath, String outputPath, String targetEntity) throws Exception {
-		SparkAtomicActionScoreJob.main(
+		SparkAtomicActionScoreJob
-			new String[] {
+			.main(
+				new String[] {
 					"-isSparkSessionManaged", Boolean.FALSE.toString(),
 					"-inputPath", inputPath,
 					"-outputPath", outputPath,
 					"-targetEntity", targetEntity,
-			}
+				});
-		);
 	}
+
 	@Test
 	void testResultScores() throws Exception {
 		final String targetEntity = RESULT;
@@ -149,8 +150,8 @@ public class SparkAtomicActionScoreJobTest {
 	void testProjectScores() throws Exception {
 		String targetEntity = PROJECT;
 		String inputResultScores = getClass()
-				.getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/project_bip_scores.json")
+			.getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/project_bip_scores.json")
-				.getPath();
+			.getPath();
 		String outputPath = workingDir.toString() + "/" + targetEntity + "/actionSet";
 
 		// execute the job to generate the action sets for project scores
@@ -159,9 +160,9 @@ public class SparkAtomicActionScoreJobTest {
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Project> projects = sc
-				.sequenceFile(outputPath, Text.class, Text.class)
+			.sequenceFile(outputPath, Text.class, Text.class)
-				.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
+			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
-				.map(aa -> ((Project) aa.getPayload()));
+			.map(aa -> ((Project) aa.getPayload()));
 
 		// test the number of projects
 		assertEquals(4, projects.count());
@@ -171,7 +172,8 @@ public class SparkAtomicActionScoreJobTest {
 		// count that the project with id testProjectId is present
 		assertEquals(1, projects.filter(row -> row.getId().equals(testProjectId)).count());
 
-		projects.filter(row -> row.getId().equals(testProjectId))
+		projects
+			.filter(row -> row.getId().equals(testProjectId))
 			.flatMap(r -> r.getMeasures().iterator())
 			.foreach(m -> {
 				log.info(m.getId() + " " + m.getUnit());
@@ -184,7 +186,7 @@ public class SparkAtomicActionScoreJobTest {
 				// ensure that the correct key is provided, i.e. score
 				assertEquals("score", kv.getKey());
 
-				switch(m.getId()) {
+				switch (m.getId()) {
 					case "numOfInfluentialResults":
 						assertEquals("0", kv.getValue());
 						break;

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java

@@ -92,7 +92,7 @@ public class PrepareH2020ProgrammeTest {
 
 		Assertions.assertEquals(0, verificationDataset.filter("classification = ''").count());
 
-		//tmp.foreach(csvProgramme -> System.out.println(OBJECT_MAPPER.writeValueAsString(csvProgramme)));
+		// tmp.foreach(csvProgramme -> System.out.println(OBJECT_MAPPER.writeValueAsString(csvProgramme)));
 
 		Assertions
 			.assertEquals(

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsTest.java

@@ -98,7 +98,7 @@ public class ReadProjectsTest {
 		Assertions.assertEquals("H2020-EU.1.3.", project.getLegalBasis());
 		Assertions.assertEquals("MSCA-IF-2019", project.getTopics());
 
-		//tmp.foreach(p -> System.out.println(OBJECT_MAPPER.writeValueAsString(p)));
+		// tmp.foreach(p -> System.out.println(OBJECT_MAPPER.writeValueAsString(p)));
 
 	}
 }

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/ReadTopicTest.java

@@ -93,7 +93,7 @@ public class ReadTopicTest {
 		Assertions.assertEquals("Individual Fellowships", topic.getTitle());
 		Assertions.assertEquals("MSCA-IF-2019", topic.getTopic());
 
-		//tmp.foreach(p -> System.out.println(OBJECT_MAPPER.writeValueAsString(p)));
+		// tmp.foreach(p -> System.out.println(OBJECT_MAPPER.writeValueAsString(p)));
 
 	}
 }

dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/job.properties

@@ -97,7 +97,7 @@ oozie.wf.application.path=${oozieTopWfApplicationPath}
 
 
 # Path where the final output should be?
-actionSetOutputPath=${workingDir}/bip_actionsets/
+actionSetOutputPath=${workingDir}/bip_actionsets
 
 # The directory to store project impact indicators
 projectImpactIndicatorsOutput=${workingDir}/project_indicators

dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml

@@ -13,7 +13,6 @@
 	</global>
 
 	<!-- start using a decision node, so as to determine from which point onwards a job will continue -->
-	<!-- <start to="get-doi-synonyms" /> -->
 	<start to="entry-point-decision" />
 
 	<decision name="entry-point-decision">
@@ -24,40 +23,28 @@
 			<case to="spark-impulse">${wf:conf('resume') eq "impulse"}</case>
 			<case to="spark-pagerank">${wf:conf('resume') eq "pagerank"}</case>
 			<case to="spark-attrank">${wf:conf('resume') eq "attrank"}</case>
-			<!-- Removed for serafeim -->
 			<!-- <case to="iterative-rankings">${wf:conf('resume') eq "rankings-iterative"}</case> -->
 			<case to="get-file-names">${wf:conf('resume') eq "format-results"}</case>
 			<case to="map-openaire-to-doi">${wf:conf('resume') eq "map-ids"}</case>
 			<case to="map-scores-to-dois">${wf:conf('resume') eq "map-scores"}</case>
 			<case to="create-openaire-ranking-graph">${wf:conf('resume') eq "start"}</case>
-			<case to="project-impact-indicators">${wf:conf('resume') eq "projects-impact"}</case>
 
-			<!-- TODO: add action set creation here -->
+			<!-- Aggregation of impact scores on the project level		-->
+			<case to="project-impact-indicators">${wf:conf('resume') eq "projects-impact"}</case>
+			<case to="create-actionset-for-projects">${wf:conf('resume') eq "projects-impact-actionsets"}</case>
+
 			<default to="create-openaire-ranking-graph" />
 		</switch>
 	</decision>
 
-	<!-- Script here written by Serafeim: maps openaire ids to their synonyms -->
+	<!-- initial step: create citation network -->
 	<action name="create-openaire-ranking-graph">
-		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
 
-			<!-- Delete previously created doi synonym folder -->
-			<!-- I think we don't need this given we don't have synonyms anymore
-			<prepare>
-				<delete path="${synonymFolder}"/>
-			</prepare>
-			-->
-
-			<!-- using configs from an example on openaire -->
 			<master>yarn-cluster</master>
 			<mode>cluster</mode>
-
+			<name>OpenAIRE Ranking Graph Creation</name>
-			<!-- This is the name of our job -->
-			<name>Openaire Ranking Graph Creation</name>
-			<!-- Script name goes here -->
 			<jar>create_openaire_ranking_graph.py</jar>
-			<!-- spark configuration options: I've taken most of them from an example from dhp workflows / Master value stolen from sandro -->
 
 			<spark-opts>
 				--executor-memory=${sparkHighExecutorMemory}
@@ -79,39 +66,30 @@
 			<arg>${sparkShufflePartitions}</arg>
 			<!-- The output of the graph should be the openaire input graph for ranking-->
 			<arg>${openaireGraphInputPath}</arg>
-			<!-- This needs to point to the file on the hdfs i think -->
+
 			<file>${wfAppPath}/create_openaire_ranking_graph.py#create_openaire_ranking_graph.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
 		<ok to="non-iterative-rankings" />
-		<!-- Go there if we have an error -->
 		<error to="openaire-graph-error" />
 
 	</action>
 
 	<!-- Citation Count and RAM are calculated in parallel-->
-	<!-- Impulse Requires resources and will be run after-->
 	<fork name="non-iterative-rankings">
 		<path start="spark-cc"/>
 		<!-- <path start="spark-impulse"/> -->
 		<path start="spark-ram"/>
 	</fork>
 
-	<!-- CC here -->
+	<!-- Run Citation Count calculation -->
 	<action name="spark-cc">
-		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
 
-			<!-- using configs from an example on openaire -->
 			<master>yarn-cluster</master>
 			<mode>cluster</mode>
-
+			<name>Citation Count calculation</name>
-			<!-- This is the name of our job -->
-			<name>Spark CC</name>
-			<!-- Script name goes here -->
 			<jar>CC.py</jar>
-			<!-- spark configuration options: I've taken most of them from an example from dhp workflows / Master value stolen from sandro -->
 
 			<spark-opts>
 				--executor-memory=${sparkHighExecutorMemory}
@@ -128,31 +106,23 @@
 			<arg>${openaireGraphInputPath}</arg>
 			<!-- number of partitions to be used on joins -->
 			<arg>${sparkShufflePartitions}</arg>
-			<!-- This needs to point to the file on the hdfs i think -->
+
 			<file>${wfAppPath}/bip-ranker/CC.py#CC.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
 		<ok to="join-non-iterative-rankings" />
-		<!-- Go there if we have an error -->
 		<error to="cc-fail" />
 
 	</action>
 
-	<!-- IMPULSE here -->
+	<!-- RAM calculation -->
 	<action name="spark-ram">
-		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
 
-			<!-- using configs from an example on openaire -->
 			<master>yarn-cluster</master>
 			<mode>cluster</mode>
-
+			<name>RAM calculation</name>
-			<!-- This is the name of our job -->
-			<name>Spark RAM</name>
-			<!-- Script name goes here -->
 			<jar>TAR.py</jar>
-			<!-- spark configuration options: I've taken most of them from an example from dhp workflows / Master value stolen from sandro -->
 
 			<spark-opts>
 				--executor-memory=${sparkHighExecutorMemory}
@@ -170,37 +140,27 @@
 			<arg>${ramGamma}</arg>
 			<arg>${currentYear}</arg>
 			<arg>RAM</arg>
-			<!-- number of partitions to be used on joins -->
 			<arg>${sparkShufflePartitions}</arg>
 			<arg>${checkpointDir}</arg>
-			<!-- This needs to point to the file on the hdfs i think -->
+
 			<file>${wfAppPath}/bip-ranker/TAR.py#TAR.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
 		<ok to="join-non-iterative-rankings" />
-		<!-- Go there if we have an error -->
 		<error to="ram-fail" />
 
 	</action>
 
-	<!-- JOIN NON-ITERATIVE METHODS AND THEN CONTINUE TO ITERATIVE ONES -->
+	<!-- Join non-iterative methods -->
 	<join name="join-non-iterative-rankings" to="spark-impulse"/>
 
-	<!-- IMPULSE here -->
 	<action name="spark-impulse">
-		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
 
-			<!-- using configs from an example on openaire -->
 			<master>yarn-cluster</master>
 			<mode>cluster</mode>
-
+			<name>Impulse calculation</name>
-			<!-- This is the name of our job -->
-			<name>Spark Impulse</name>
-			<!-- Script name goes here -->
 			<jar>CC.py</jar>
-			<!-- spark configuration options: I've taken most of them from an example from dhp workflows / Master value stolen from sandro -->
 
 			<spark-opts>
 				--executor-memory=${sparkHighExecutorMemory}
@@ -218,47 +178,22 @@
 			<!-- number of partitions to be used on joins -->
 			<arg>${sparkShufflePartitions}</arg>
 			<arg>3</arg>
-			<!-- This needs to point to the file on the hdfs i think -->
+
 			<file>${wfAppPath}/bip-ranker/CC.py#CC.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
 		<ok to="spark-pagerank" />
-		<!-- Go there if we have an error -->
 		<error to="impulse-fail" />
 
 	</action>
 
-	<!-- Removed for ser to make pagerank & attrank serial -->
-	<!-- 
-	<fork name="iterative-rankings">
-		<path start="spark-pagerank"/>
-		<path start="spark-attrank"/>
-	</fork>
-	-->
-
-	<!-- PAGERANK here -->
 	<action name="spark-pagerank">
-		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
 
-			<!-- we could add map-reduce configs here, but I don't know if we need them -->
-			<!-- This is the type of master-client configuration for running spark -->
-			<!-- <master>yarn-client</master> -->
-			<!-- Reference says: The master element indicates the url of the Spark Master. Ex: spark://host:port, mesos://host:port, yarn-cluster, yarn-master, or local. -->
-			<!-- <master>local[*]</master> -->
-			<!-- Reference says: The mode element if present indicates the mode of spark, where to run spark driver program. Ex: client,cluster. | In my case I always have a client -->
-			<!-- <mode>client</mode> -->
-
-			<!-- using configs from an example on openaire -->
 			<master>yarn-cluster</master>
 			<mode>cluster</mode>
-
+			<name>Pagerank calculation</name>
-			<!-- This is the name of our job -->
-			<name>Spark Pagerank</name>
-			<!-- Script name goes here -->
 			<jar>PageRank.py</jar>
-			<!-- spark configuration options: I've taken most of them from an example from dhp workflows / Master value stolen from sandro -->
 
 			<spark-opts>
 				--executor-memory=${sparkHighExecutorMemory}
@@ -279,31 +214,22 @@
 			<!-- number of partitions to be used on joins -->
 			<arg>${sparkShufflePartitions}</arg>
 			<arg>dfs</arg>
-			<!-- This needs to point to the file on the hdfs i think -->
+
 			<file>${wfAppPath}/bip-ranker/PageRank.py#PageRank.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
 		<ok to="spark-attrank" />
-		<!-- Go there if we have an error -->
 		<error to="pagerank-fail" />
 
 	</action>
 
-	<!-- ATTRANK here -->
 	<action name="spark-attrank">
-		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
 
-			<!-- using configs from an example on openaire -->
 			<master>yarn-cluster</master>
 			<mode>cluster</mode>
-
+			<name>AttRank calculation</name>
-			<!-- This is the name of our job -->
-			<name>Spark AttRank</name>
-			<!-- Script name goes here -->
 			<jar>AttRank.py</jar>
-			<!-- spark configuration options: I've taken most of them from an example from dhp workflows / Master value stolen from sandro -->
 
 			<spark-opts>
 				--executor-memory=${sparkHighExecutorMemory}
@@ -329,27 +255,16 @@
 			<!-- number of partitions to be used on joins -->
 			<arg>${sparkShufflePartitions}</arg>
 			<arg>dfs</arg>
-			<!-- This needs to point to the file on the hdfs i think -->
+
 			<file>${wfAppPath}/bip-ranker/AttRank.py#AttRank.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
 		<ok to="get-file-names" />
-		<!-- Go there if we have an error -->
 		<error to="attrank-fail" />
 
 	</action>
 
-	<!-- Removed for ser -->
-	<!--
-	 JOIN ITERATIVE METHODS AND THEN END =
-	<join name="join-iterative-rankings" to="end" />
-	to="get-file-names"/>  
-	-->
-
-	<!-- This will be a shell action that will output key-value pairs for output files -->
 	<action name="get-file-names">
-		<!-- This is required as a tag for shell jobs -->
 		<shell xmlns="uri:oozie:shell-action:0.3">
 
 			<!-- Exec is needed for shell commands - points to type of shell command -->
@@ -359,15 +274,12 @@
 			<!-- We only pass the directory where we expect to find the rankings -->
 			<argument>${workingDir}</argument>
 
-			<!-- the name of the file run -->
 			<file>${wfAppPath}/get_ranking_files.sh#get_ranking_files.sh</file>
 			<!-- Get the output in order to be usable by following actions -->
 			<capture-output/>
 		</shell>
 
-		<!-- Do this after finishing okay -->
 		<ok to="format-result-files" />
-		<!-- Go there if we have an error -->
 		<error to="filename-getting-error" />
 
 	</action>
@@ -382,18 +294,12 @@
 	<!-- Format json files -->
 	<!-- Two parts: a) format files b) make the file endings .json.gz -->
 	<action name="format-json-files">
-		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
 
-			<!-- using configs from an example on openaire -->
 			<master>yarn-cluster</master>
 			<mode>cluster</mode>
-
-			<!-- This is the name of our job -->
 			<name>Format Ranking Results JSON</name>
-			<!-- Script name goes here -->
 			<jar>format_ranking_results.py</jar>
-			<!-- spark configuration options: I've taken most of them from an example from dhp workflows / Master value stolen from sandro -->
 
 			<spark-opts>
 				--executor-memory=${sparkNormalExecutorMemory}
@@ -418,13 +324,11 @@
 			<arg>${sparkShufflePartitions}</arg>
 			<!-- Type of data to be produced [bip (dois) / openaire (openaire-ids) ] -->
 			<arg>openaire</arg>
-			<!-- This needs to point to the file on the hdfs i think -->
+
 			<file>${wfAppPath}/format_ranking_results.py#format_ranking_results.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
 		<ok to="join-file-formatting" />
-		<!-- Go there if we have an error -->
 		<error to="json-formatting-fail" />
 	</action>
 
@@ -470,18 +374,15 @@
 			<file>${wfAppPath}/format_ranking_results.py#format_ranking_results.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
 		<ok to="join-file-formatting" />
-		<!-- Go there if we have an error -->
 		<error to="bip-formatting-fail" />
 	</action>
 
-	<!-- Finish formatting data and end -->
+	<!-- Finish formatting jobs -->
 	<join name="join-file-formatting" to="map-openaire-to-doi"/>
 
-	<!-- Script here written by Serafeim: maps openaire ids to their synonyms -->
+	<!-- maps openaire ids to DOIs -->
 	<action name="map-openaire-to-doi">
-		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
 
 			<!-- Delete previously created doi synonym folder -->
@@ -489,15 +390,10 @@
 				<delete path="${synonymFolder}"/>
 			</prepare>
 
-			<!-- using configs from an example on openaire -->
 			<master>yarn-cluster</master>
 			<mode>cluster</mode>
-
-			<!-- This is the name of our job -->
 			<name>Openaire-DOI synonym collection</name>
-			<!-- Script name goes here -->
 			<jar>map_openaire_ids_to_dois.py</jar>
-			<!-- spark configuration options: I've taken most of them from an example from dhp workflows / Master value stolen from sandro -->
 
 			<spark-opts>
 				--executor-memory=${sparkHighExecutorMemory}
@@ -514,19 +410,16 @@
 			<arg>${openaireDataInput}/</arg>
 			<!-- number of partitions to be used on joins -->
 			<arg>${synonymFolder}</arg>
-			<!-- This needs to point to the file on the hdfs i think -->
+
 			<file>${wfAppPath}/map_openaire_ids_to_dois.py#map_openaire_ids_to_dois.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
 		<ok to="map-scores-to-dois" />
-		<!-- Go there if we have an error -->
 		<error to="synonym-collection-fail" />
 
 	</action>
 
-
+	<!-- mapping openaire scores to DOIs -->
-	<!-- Script here written by Serafeim: maps openaire ids to their synonyms -->
 	<action name="map-scores-to-dois">
 		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
@@ -534,12 +427,8 @@
 			<!-- using configs from an example on openaire -->
 			<master>yarn-cluster</master>
 			<mode>cluster</mode>
-
-			<!-- This is the name of our job -->
 			<name>Mapping Openaire Scores to DOIs</name>
-			<!-- Script name goes here -->
 			<jar>map_scores_to_dois.py</jar>
-			<!-- spark configuration options: I've taken most of them from an example from dhp workflows / Master value stolen from sandro -->
 
 			<spark-opts>
 				--executor-memory=${sparkHighExecutorMemory}
@@ -563,19 +452,16 @@
 			<arg>${nameNode}/${workingDir}/${wf:actionData('get-file-names')['impulse_file']}</arg>
 			<arg>${nameNode}/${workingDir}/${wf:actionData('get-file-names')['ram_file']}</arg>
 
-			<!-- This needs to point to the file on the hdfs i think -->
 			<file>${wfAppPath}/map_scores_to_dois.py#map_scores_to_dois.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
+		<ok to="delete-output-path-for-actionset" />
-		<ok to="deleteOutputPathForActionSet" />
-		<!-- This is the initial code <ok to="deleteOutputPathForActionSet" /> -->
-		<!-- Go there if we have an error -->
 		<error to="map-scores-fail" />
 
 	</action>
 
-	<action name="deleteOutputPathForActionSet">
+	<!-- Re-create folder for result and project actionsets -->
+	<action name="delete-output-path-for-actionset">
 		<fs>
 			<delete path="${actionSetOutputPath}/results/"/>
 			<delete path="${actionSetOutputPath}/projects/"/>
@@ -583,17 +469,19 @@
 			<mkdir path="${actionSetOutputPath}/results/"/>
 			<mkdir path="${actionSetOutputPath}/projects/"/>
 		</fs>
-		<ok to="createActionSetForResults"/>
+		<ok to="create-actionset-for-results"/>
 		<error to="actionset-delete-fail"/>
 	</action>
 
-	<action name="createActionSetForResults">
+	<action name="create-actionset-for-results">
 		<spark xmlns="uri:oozie:spark-action:0.2">
-			<master>yarn</master>
+
+			<master>yarn-cluster</master>
 			<mode>cluster</mode>
 			<name>Produces the atomic action with the bip finder scores for publications</name>
 			<class>eu.dnetlib.dhp.actionmanager.bipfinder.SparkAtomicActionScoreJob</class>
 			<jar>dhp-aggregation-${projectVersion}.jar</jar>
+
 			<spark-opts>
 				--executor-memory=${sparkNormalExecutorMemory}
 				--executor-cores=${sparkExecutorCores}
@@ -608,23 +496,19 @@
 			<arg>--outputPath</arg><arg>${actionSetOutputPath}/results/</arg>
 			<arg>--targetEntity</arg><arg>result</arg>
 		</spark>
+
 		<ok to="project-impact-indicators"/>
 		<error to="actionset-creation-fail"/>
+
 	</action>
 
 	<action name="project-impact-indicators">
-		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
 
-			<!-- using configs from an example on openaire -->
 			<master>yarn-cluster</master>
 			<mode>cluster</mode>
-
+			<name>Project Impact Indicators calculation</name>
-			<!-- This is the name of our job -->
-			<name>Project Impact Indicators</name>
-			<!-- Script name goes here -->
 			<jar>projects_impact.py</jar>
-			<!-- spark configuration options: I've taken most of them from an example from dhp workflows / Master value stolen from sandro -->
 
 			<spark-opts>
 				--executor-memory=${sparkHighExecutorMemory}
@@ -638,9 +522,8 @@
 			</spark-opts>
 
 			<!-- Script arguments here -->
-
 			<!-- graph data folder from which to read relations -->
-			<arg>${openaireDataInput}/relations</arg>
+			<arg>${openaireDataInput}/relation</arg>
 
 			<!-- input files with impact indicators for results	-->
 			<arg>${nameNode}/${workingDir}/${wf:actionData('get-file-names')['pr_file']}</arg>
@@ -652,26 +535,23 @@
 			<arg>${sparkShufflePartitions}</arg>
 
 			<arg>${projectImpactIndicatorsOutput}</arg>
-
-			<!-- This needs to point to the file on the hdfs i think -->
 			<file>${wfAppPath}/projects_impact.py#projects_impact.py</file>
 		</spark>
 
-		<!-- Do this after finishing okay -->
+		<ok to="create-actionset-for-projects" />
-		<ok to="createActionSetForProjects" />
-
-		<!-- Go there if we have an error -->
 		<error to="project-impact-indicators-fail" />
 
 	</action>
 
-	<action name="createActionSetForProjects">
+	<action name="create-actionset-for-projects">
 		<spark xmlns="uri:oozie:spark-action:0.2">
-			<master>yarn</master>
+
+			<master>yarn-cluster</master>
 			<mode>cluster</mode>
 			<name>Produces the atomic action with the bip finder scores for projects</name>
 			<class>eu.dnetlib.dhp.actionmanager.bipfinder.SparkAtomicActionScoreJob</class>
 			<jar>dhp-aggregation-${projectVersion}.jar</jar>
+
 			<spark-opts>
 				--executor-memory=${sparkNormalExecutorMemory}
 				--executor-cores=${sparkExecutorCores}
@@ -682,12 +562,15 @@
 				--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
 				--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
 			</spark-opts>
+
 			<arg>--inputPath</arg><arg>${projectImpactIndicatorsOutput}</arg>
 			<arg>--outputPath</arg><arg>${actionSetOutputPath}/projects/</arg>
 			<arg>--targetEntity</arg><arg>project</arg>
 		</spark>
+
 		<ok to="end"/>
 		<error to="actionset-project-creation-fail"/>
+
 	</action>
 
 	<!-- Definitions of failure messages -->