sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

changed behavior in transformation job, that doesn't fail at first error

This commit is contained in:
Sandro La Bruzzo 2021-09-07 19:05:46 +02:00
parent 3c6fc2096c
commit aed29156c7
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java
View File

@ -7,6 +7,7 @@ import static eu.dnetlib.dhp.utils.DHPUtils.*;
import java.io.IOException; import java.io.IOException;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
@ -126,7 +127,8 @@ public class TransformSparkJobNode {
JavaRDD<MetadataRecord> mdstore = inputMDStore JavaRDD<MetadataRecord> mdstore = inputMDStore
.javaRDD() .javaRDD()
.repartition(getRepartitionNumber(totalInput, rpt)) .repartition(getRepartitionNumber(totalInput, rpt))
.map((Function<MetadataRecord, MetadataRecord>) x::call); .map((Function<MetadataRecord, MetadataRecord>) x::call)
.filter((Function<MetadataRecord, Boolean>) Objects::nonNull);
saveDataset(spark.createDataset(mdstore.rdd(), encoder), outputBasePath + MDSTORE_DATA_PATH); saveDataset(spark.createDataset(mdstore.rdd(), encoder), outputBasePath + MDSTORE_DATA_PATH);
log.info("Transformed item {}", ct.getProcessedItems().count()); log.info("Transformed item {}", ct.getProcessedItems().count());

3
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/XSLTTransformationFunction.java
View File

@ -81,7 +81,8 @@ public class XSLTTransformationFunction implements MapFunction<MetadataRecord, M
return value; return value;
} catch (Throwable e) { } catch (Throwable e) {
aggregationCounter.getErrorItems().add(1); aggregationCounter.getErrorItems().add(1);
throw new RuntimeException(e); return null;
// throw new RuntimeException(e);
} }
} }