[aggregator graph] save invalid records aside for further inspection

discard-non-wellformed
Claudio Atzori 2 years ago
parent 9e7ec4198f
commit 1e42d984e1

@ -143,7 +143,7 @@ public abstract class AbstractMdRecordToOafMapper {
return createOafs(doc, type, instances, collectedFrom, info, lastUpdateTimestamp);
} catch (DocumentException e) {
log.error("Error with record:\n" + xml);
return null;
return Lists.newArrayList();
}
}

@ -150,8 +150,8 @@ public class GenerateEntitiesApplication {
.sequenceFile(sp, Text.class, Text.class)
.map(k -> new Tuple2<>(k._1().toString(), k._2().toString()))
.map(k -> convertToListOaf(k._1(), k._2(), shouldHashId, vocs))
.filter(Objects::nonNull)
.flatMap(List::iterator));
.flatMap(List::iterator)
.filter(Objects::nonNull));
}
switch (mode) {
@ -225,7 +225,8 @@ public class GenerateEntitiesApplication {
final boolean shouldHashId,
final VocabularyGroup vocs) {
if (Objects.isNull(convertToListOaf(id, s, shouldHashId, vocs))) {
final List<Oaf> oaf = convertToListOaf(id, s, shouldHashId, vocs);
if (Optional.ofNullable(oaf).map(List::isEmpty).orElse(false)) {
return s;
}
return null;
@ -235,8 +236,7 @@ public class GenerateEntitiesApplication {
try {
return OBJECT_MAPPER.readValue(s, clazz);
} catch (final Exception e) {
log.error("Error parsing object of class: {}", clazz);
log.error(s);
log.error("Error parsing object of class: {}:\n{}", clazz, s);
throw new IllegalArgumentException(e);
}
}

@ -926,11 +926,12 @@ class MappersTest {
}
@Test
void testNotWellFormed() throws IOException, DocumentException {
void testNotWellFormed() throws IOException {
final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_notwellformed.xml")));
assertEquals(null, new OafToOafMapper(vocs, false, true).processMdRecord(xml));
final List<Oaf> actual = new OafToOafMapper(vocs, false, true).processMdRecord(xml);
assertNotNull(actual);
assertTrue(actual.isEmpty());
}
private void assertValidId(final String id) {

Loading…
Cancel
Save