fix issue with flatMap - the return type must not be null

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java

@@ -75,17 +75,27 @@ public class PrepareProjects {
 		HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration());
 	}
 
-	private static void exec(SparkSession spark, String progjectPath, String dbProjectPath, String outputPath) {
-		Dataset<CSVProject> project = readPath(spark, progjectPath, CSVProject.class);
+	private static void exec(SparkSession spark, String projectPath, String dbProjectPath, String outputPath) {
+		Dataset<CSVProject> project = readPath(spark, projectPath, CSVProject.class);
 		Dataset<ProjectSubset> dbProjects = readPath(spark, dbProjectPath, ProjectSubset.class);
 
-		dbProjects.joinWith(project, dbProjects.col("code").equalTo(project.col("id")), "left")
-				.flatMap((FlatMapFunction<Tuple2<ProjectSubset, CSVProject>, CSVProject>) value -> {
-					Optional<CSVProject> csvProject = Optional.ofNullable(value._2());
-					if(! csvProject.isPresent()){
-						return null;
+		dbProjects
+			.joinWith(project, dbProjects.col("code").equalTo(project.col("id")), "left")
+			.flatMap(getTuple2CSVProjectFlatMapFunction(), Encoders.bean(CSVProject.class))
+			.filter(Objects::nonNull)
+			.write()
+			.mode(SaveMode.Overwrite)
+			.option("compression", "gzip")
+			.json(outputPath);
+
 	}
+
+	private static FlatMapFunction<Tuple2<ProjectSubset, CSVProject>, CSVProject> getTuple2CSVProjectFlatMapFunction() {
+		return (FlatMapFunction<Tuple2<ProjectSubset, CSVProject>, CSVProject>) value -> {
+			Optional<CSVProject> csvProject = Optional.ofNullable(value._2());
 			List<CSVProject> csvProjectList = new ArrayList<>();
+			if (csvProject.isPresent()) {
+
 				String[] programme = csvProject.get().getProgramme().split(";");
 				Arrays
 					.stream(programme)
@@ -95,36 +105,9 @@ public class PrepareProjects {
 						proj.setId(csvProject.get().getId());
 						csvProjectList.add(proj);
 					});
-
+			}
 			return csvProjectList.iterator();
-				}, Encoders.bean(CSVProject.class))
-		.write()
-		.mode(SaveMode.Overwrite)
-		.option("compression", "gzip")
-		.json(outputPath);
-//
-//				.map(value -> {
-//					Optional<CSVProject> csvProject = Optional.ofNullable(value._2());
-//				}, Encoders.bean(CSVProject.class))
-//		.filter(Objects::nonNull)
-//			.toJavaRDD()
-//			.flatMap(p -> {
-//				List<CSVProject> csvProjectList = new ArrayList<>();
-//				String[] programme = p.getProgramme().split(";");
-//				Arrays
-//					.stream(programme)
-//					.forEach(value -> {
-//						CSVProject csvProject = new CSVProject();
-//						csvProject.setProgramme(value);
-//						csvProject.setId(p.getId());
-//						csvProjectList.add(csvProject);
-//					});
-//
-//				return csvProjectList.iterator();
-//			})
-//			.map(p -> OBJECT_MAPPER.writeValueAsString(p))
-//			.saveAsTextFile(outputPath);
-
+		};
 	}
 
 	public static <R> Dataset<R> readPath(