Fixes in workflow.xml

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java

@@ -9,7 +9,6 @@ import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipProjectModel;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
@@ -25,8 +24,9 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipResultModel;
 import eu.dnetlib.dhp.actionmanager.bipmodel.BipScore;
+import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipProjectModel;
+import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipResultModel;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
 import eu.dnetlib.dhp.schema.action.AtomicAction;
@@ -89,8 +89,7 @@ public class SparkAtomicActionScoreJob implements Serializable {
 					default:
 						throw new RuntimeException("Unknown target entity: " + targetEntity);
 				}
-			}
+			});
-		);
 	}
 
 	private static <I extends Project> void prepareProjects(SparkSession spark, String inputPath, String outputPath) {
@@ -106,7 +105,8 @@ public class SparkAtomicActionScoreJob implements Serializable {
 		}, Encoders.bean(Project.class))
 			.toJavaRDD()
 			.map(p -> new AtomicAction(Project.class, p))
-		.mapToPair( aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
+			.mapToPair(
+				aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
 					new Text(OBJECT_MAPPER.writeValueAsString(aa))))
 			.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class);
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipmodel/score/deserializers/BipProjectModel.java

@@ -1,21 +1,23 @@
+
 package eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers;
 
-import com.opencsv.bean.CsvBindByPosition;
+import static eu.dnetlib.dhp.actionmanager.Constants.*;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
-import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import eu.dnetlib.dhp.schema.oaf.Measure;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import static eu.dnetlib.dhp.actionmanager.Constants.*;
+import com.opencsv.bean.CsvBindByPosition;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Measure;
+import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
 
 @NoArgsConstructor
 @AllArgsConstructor
@@ -38,32 +40,35 @@ public class BipProjectModel {
 		KeyValue kv = new KeyValue();
 		kv.setKey("score");
 		kv.setValue(measureValue);
-        kv.setDataInfo(
+		kv
-            OafMapperUtils.dataInfo(
+			.setDataInfo(
+				OafMapperUtils
+					.dataInfo(
 						false,
 						UPDATE_DATA_INFO_TYPE,
 						true,
 						false,
-                OafMapperUtils.qualifier(
+						OafMapperUtils
+							.qualifier(
 								UPDATE_MEASURE_BIP_CLASS_ID,
 								UPDATE_CLASS_NAME,
 								ModelConstants.DNET_PROVENANCE_ACTIONS,
 								ModelConstants.DNET_PROVENANCE_ACTIONS),
-    "")
+						""));
-        );
 
 		Measure measure = new Measure();
 		measure.setId(measureId);
 		measure.setUnit(Collections.singletonList(kv));
 		return measure;
 	}
+
 	public List<Measure> toMeasures() {
-        return Arrays.asList(
+		return Arrays
+			.asList(
 				createMeasure("numOfInfluentialResults", numOfInfluentialResults),
 				createMeasure("numOfPopularResults", numOfPopularResults),
 				createMeasure("totalImpulse", totalImpulse),
-            createMeasure("totalCitationCount", totalCitationCount)
+				createMeasure("totalCitationCount", totalCitationCount));
-        );
 	}
 
 }

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipmodel/score/deserializers/BipResultModel.java

@@ -1,13 +1,13 @@
 
 package eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers;
 
-import eu.dnetlib.dhp.actionmanager.bipmodel.Score;
-
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 
+import eu.dnetlib.dhp.actionmanager.bipmodel.Score;
+
 /**
  * Class that maps the model of the bipFinder! input data.
  * Only needed for deserialization purposes

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java

@@ -24,8 +24,8 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipResultModel;
 import eu.dnetlib.dhp.actionmanager.bipmodel.BipScore;
+import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipResultModel;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
 import eu.dnetlib.dhp.schema.common.ModelConstants;

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJobTest.java

@@ -7,8 +7,6 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
-import eu.dnetlib.dhp.schema.oaf.Project;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.spark.SparkConf;
@@ -28,6 +26,8 @@ import org.slf4j.LoggerFactory;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import eu.dnetlib.dhp.schema.action.AtomicAction;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Project;
 import eu.dnetlib.dhp.schema.oaf.Result;
 
 public class SparkAtomicActionScoreJobTest {
@@ -73,15 +73,16 @@ public class SparkAtomicActionScoreJobTest {
 	}
 
 	private void runJob(String inputPath, String outputPath, String targetEntity) throws Exception {
-		SparkAtomicActionScoreJob.main(
+		SparkAtomicActionScoreJob
+			.main(
 				new String[] {
 					"-isSparkSessionManaged", Boolean.FALSE.toString(),
 					"-inputPath", inputPath,
 					"-outputPath", outputPath,
 					"-targetEntity", targetEntity,
+				});
 	}
-		);
+
-	}
 	@Test
 	void testResultScores() throws Exception {
 		final String targetEntity = RESULT;
@@ -171,7 +172,8 @@ public class SparkAtomicActionScoreJobTest {
 		// count that the project with id testProjectId is present
 		assertEquals(1, projects.filter(row -> row.getId().equals(testProjectId)).count());
 
-		projects.filter(row -> row.getId().equals(testProjectId))
+		projects
+			.filter(row -> row.getId().equals(testProjectId))
 			.flatMap(r -> r.getMeasures().iterator())
 			.foreach(m -> {
 				log.info(m.getId() + " " + m.getUnit());

dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/job.properties

@@ -97,7 +97,7 @@ oozie.wf.application.path=${oozieTopWfApplicationPath}
 
 
 # Path where the final output should be?
-actionSetOutputPath=${workingDir}/bip_actionsets/
+actionSetOutputPath=${workingDir}/bip_actionsets
 
 # The directory to store project impact indicators
 projectImpactIndicatorsOutput=${workingDir}/project_indicators

dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml

@@ -24,20 +24,21 @@
 			<case to="spark-impulse">${wf:conf('resume') eq "impulse"}</case>
 			<case to="spark-pagerank">${wf:conf('resume') eq "pagerank"}</case>
 			<case to="spark-attrank">${wf:conf('resume') eq "attrank"}</case>
-			<!-- Removed for serafeim -->
 			<!-- <case to="iterative-rankings">${wf:conf('resume') eq "rankings-iterative"}</case> -->
 			<case to="get-file-names">${wf:conf('resume') eq "format-results"}</case>
 			<case to="map-openaire-to-doi">${wf:conf('resume') eq "map-ids"}</case>
 			<case to="map-scores-to-dois">${wf:conf('resume') eq "map-scores"}</case>
 			<case to="create-openaire-ranking-graph">${wf:conf('resume') eq "start"}</case>
-			<case to="project-impact-indicators">${wf:conf('resume') eq "projects-impact"}</case>
 
-			<!-- TODO: add action set creation here -->
+			<!-- Aggregation of impact scores on the project level		-->
+			<case to="project-impact-indicators">${wf:conf('resume') eq "projects-impact"}</case>
+			<case to="create-actionset-for-projects">${wf:conf('resume') eq "projects-impact-actionsets"}</case>
+
 			<default to="create-openaire-ranking-graph" />
 		</switch>
 	</decision>
 
-	<!-- Script here written by Serafeim: maps openaire ids to their synonyms -->
+	<!-- maps openaire ids to their synonyms -->
 	<action name="create-openaire-ranking-graph">
 		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
@@ -479,7 +480,7 @@
 	<!-- Finish formatting data and end -->
 	<join name="join-file-formatting" to="map-openaire-to-doi"/>
 
-	<!-- Script here written by Serafeim: maps openaire ids to their synonyms -->
+	<!-- maps openaire ids to their synonyms -->
 	<action name="map-openaire-to-doi">
 		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
@@ -526,7 +527,7 @@
 	</action>
 
 
-	<!-- Script here written by Serafeim: maps openaire ids to their synonyms -->
+	<!-- maps openaire ids to their synonyms -->
 	<action name="map-scores-to-dois">
 		<!-- This is required as a tag for spark jobs, regardless of programming language -->
 		<spark xmlns="uri:oozie:spark-action:0.2">
@@ -568,14 +569,14 @@
 		</spark>
 
 		<!-- Do this after finishing okay -->
-		<ok to="deleteOutputPathForActionSet" />
+		<ok to="delete-output-path-for-actionset" />
-		<!-- This is the initial code <ok to="deleteOutputPathForActionSet" /> -->
+		<!-- This is the initial code <ok to="delete-output-path-for-actionset" /> -->
 		<!-- Go there if we have an error -->
 		<error to="map-scores-fail" />
 
 	</action>
 
-	<action name="deleteOutputPathForActionSet">
+	<action name="delete-output-path-for-actionset">
 		<fs>
 			<delete path="${actionSetOutputPath}/results/"/>
 			<delete path="${actionSetOutputPath}/projects/"/>
@@ -583,13 +584,13 @@
 			<mkdir path="${actionSetOutputPath}/results/"/>
 			<mkdir path="${actionSetOutputPath}/projects/"/>
 		</fs>
-		<ok to="createActionSetForResults"/>
+		<ok to="create-actionset-for-results"/>
 		<error to="actionset-delete-fail"/>
 	</action>
 
-	<action name="createActionSetForResults">
+	<action name="create-actionset-for-results">
 		<spark xmlns="uri:oozie:spark-action:0.2">
-			<master>yarn</master>
+			<master>yarn-cluster</master>
 			<mode>cluster</mode>
 			<name>Produces the atomic action with the bip finder scores for publications</name>
 			<class>eu.dnetlib.dhp.actionmanager.bipfinder.SparkAtomicActionScoreJob</class>
@@ -640,7 +641,7 @@
 			<!-- Script arguments here -->
 
 			<!-- graph data folder from which to read relations -->
-			<arg>${openaireDataInput}/relations</arg>
+			<arg>${openaireDataInput}/relation</arg>
 
 			<!-- input files with impact indicators for results	-->
 			<arg>${nameNode}/${workingDir}/${wf:actionData('get-file-names')['pr_file']}</arg>
@@ -658,16 +659,16 @@
 		</spark>
 
 		<!-- Do this after finishing okay -->
-		<ok to="createActionSetForProjects" />
+		<ok to="create-actionset-for-projects" />
 
 		<!-- Go there if we have an error -->
 		<error to="project-impact-indicators-fail" />
 
 	</action>
 
-	<action name="createActionSetForProjects">
+	<action name="create-actionset-for-projects">
 		<spark xmlns="uri:oozie:spark-action:0.2">
-			<master>yarn</master>
+			<master>yarn-cluster</master>
 			<mode>cluster</mode>
 			<name>Produces the atomic action with the bip finder scores for projects</name>
 			<class>eu.dnetlib.dhp.actionmanager.bipfinder.SparkAtomicActionScoreJob</class>