From ba8a024af930df4dbdf75ceb2ef113dd71fcfa74 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 12 Jun 2020 10:45:11 +0200 Subject: [PATCH] avoid NPEs merging titles --- .../eu/dnetlib/dhp/schema/oaf/Result.java | 13 ++- .../oa/graph/clean/NormalizeEmptyFields.java | 88 ------------------- 2 files changed, 5 insertions(+), 96 deletions(-) delete mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java index 09e77a244..c9d0ac7c7 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java @@ -254,28 +254,25 @@ public class Result extends OafEntity implements Serializable { final StructuredProperty p = baseMainTitle; title = title.stream().filter(t -> t != p).collect(Collectors.toList()); } -// -// -// title.remove(baseMainTitle); } StructuredProperty newMainTitle = null; if (r.getTitle() != null) { newMainTitle = getMainTitle(r.getTitle()); - if (newMainTitle != null) { + if (newMainTitle != null && title != null) { final StructuredProperty p = newMainTitle; title = title.stream().filter(t -> t != p).collect(Collectors.toList()); } - - // r.getTitle().remove(newMainTitle); } - if (newMainTitle != null && compareTrust(this, r) < 0) + if (newMainTitle != null && compareTrust(this, r) < 0) { baseMainTitle = newMainTitle; + } title = mergeLists(title, r.getTitle()); - if (title != null && baseMainTitle != null) + if (title != null && baseMainTitle != null) { title.add(baseMainTitle); + } relevantdate = mergeLists(relevantdate, r.getRelevantdate()); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java deleted file mode 100644 index 77537801f..000000000 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java +++ /dev/null @@ -1,88 +0,0 @@ - -package eu.dnetlib.dhp.oa.graph.clean; - -import eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils; -import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; -import eu.dnetlib.dhp.schema.oaf.Oaf; -import eu.dnetlib.dhp.schema.oaf.Qualifier; -import org.apache.commons.lang3.StringUtils; -import org.apache.spark.api.java.function.MapFunction; - -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; - -public class NormalizeEmptyFields implements MapFunction { - - private VocabularyGroup vocabularies; - - public NormalizeEmptyFields(VocabularyGroup vocabularies) { - this.vocabularies = vocabularies; - } - - @Override - public T call(T value) throws Exception { - - doNormalize(value); - - return value; - } - - private void doNormalize(Object o) { - if (Objects.isNull(o)) { - return; - } - - if (o instanceof Iterable) { - for (Object oi : (Iterable) o) { - doNormalize(oi); - } - } else { - - Class clazz = o.getClass(); - - if (clazz.isPrimitive() - || o instanceof Integer - || o instanceof Double - || o instanceof Float - || o instanceof Long - || o instanceof Boolean - || o instanceof String) { - return; - } else { - try { - for (Field field : getAllFields(new LinkedList<>(), clazz)) { - field.setAccessible(true); - Object value = field.get(o); - if (value instanceof Qualifier && Objects.isNull(value)) { - field.set(o, OafMapperUtils.unknown("", "")); - } else if (value instanceof Field && Objects.isNull(value)) { - - } else { - doNormalize(value); - } - } - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new RuntimeException(e); - } - } - } - } - - private static List getAllFields(List fields, Class clazz) { - fields.addAll(Arrays.asList(clazz.getDeclaredFields())); - - final Class superclass = clazz.getSuperclass(); - if (Objects.nonNull(superclass) && superclass.getPackage().equals(Oaf.class.getPackage())) { - getAllFields(fields, superclass); - } - - return fields; - } - - public VocabularyGroup getVocabularies() { - return vocabularies; - } -}