From aed29156c78192ba4c2841fc6ae179c391f1b088 Mon Sep 17 00:00:00 2001
From: Sandro La Bruzzo <sandro.labruzzo@isti.cnr.it>
Date: Tue, 7 Sep 2021 19:05:46 +0200
Subject: [PATCH] changed behavior in transformation job, that  doesn't fail at
 first error

---
 .../eu/dnetlib/dhp/transformation/TransformSparkJobNode.java  | 4 +++-
 .../dhp/transformation/xslt/XSLTTransformationFunction.java   | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java
index 4fe79bf76..ed867c7f2 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java
@@ -7,6 +7,7 @@ import static eu.dnetlib.dhp.utils.DHPUtils.*;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 
 import org.apache.commons.io.IOUtils;
@@ -126,7 +127,8 @@ public class TransformSparkJobNode {
 				JavaRDD<MetadataRecord> mdstore = inputMDStore
 					.javaRDD()
 					.repartition(getRepartitionNumber(totalInput, rpt))
-					.map((Function<MetadataRecord, MetadataRecord>) x::call);
+					.map((Function<MetadataRecord, MetadataRecord>) x::call)
+					.filter((Function<MetadataRecord, Boolean>) Objects::nonNull);
 				saveDataset(spark.createDataset(mdstore.rdd(), encoder), outputBasePath + MDSTORE_DATA_PATH);
 
 				log.info("Transformed item {}", ct.getProcessedItems().count());
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/XSLTTransformationFunction.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/XSLTTransformationFunction.java
index acf48ccc5..54192a7be 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/XSLTTransformationFunction.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/XSLTTransformationFunction.java
@@ -81,7 +81,8 @@ public class XSLTTransformationFunction implements MapFunction<MetadataRecord, M
 			return value;
 		} catch (Throwable e) {
 			aggregationCounter.getErrorItems().add(1);
-			throw new RuntimeException(e);
+			return null;
+//			throw new RuntimeException(e);
 		}
 	}