andreas.czerniak
/
BrBETA_dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[aggregator graph] save invalid records aside for further inspection

This commit is contained in:
Claudio Atzori 2022-09-15 10:49:42 +02:00
parent 9e7ec4198f
commit 1e42d984e1
3 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java
View File

@ -143,7 +143,7 @@ public abstract class AbstractMdRecordToOafMapper {
return createOafs(doc, type, instances, collectedFrom, info, lastUpdateTimestamp); return createOafs(doc, type, instances, collectedFrom, info, lastUpdateTimestamp);
} catch (DocumentException e) { } catch (DocumentException e) {
log.error("Error with record:\n" + xml); log.error("Error with record:\n" + xml);
return null; return Lists.newArrayList();
} }
} }

10
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java
View File

@ -150,8 +150,8 @@ public class GenerateEntitiesApplication {
.sequenceFile(sp, Text.class, Text.class) .sequenceFile(sp, Text.class, Text.class)
.map(k -> new Tuple2<>(k._1().toString(), k._2().toString())) .map(k -> new Tuple2<>(k._1().toString(), k._2().toString()))
.map(k -> convertToListOaf(k._1(), k._2(), shouldHashId, vocs)) .map(k -> convertToListOaf(k._1(), k._2(), shouldHashId, vocs))
.filter(Objects::nonNull) .flatMap(List::iterator)
.flatMap(List::iterator)); .filter(Objects::nonNull));
} }
switch (mode) { switch (mode) {
@ -225,7 +225,8 @@ public class GenerateEntitiesApplication {
final boolean shouldHashId, final boolean shouldHashId,
final VocabularyGroup vocs) { final VocabularyGroup vocs) {
if (Objects.isNull(convertToListOaf(id, s, shouldHashId, vocs))) { final List<Oaf> oaf = convertToListOaf(id, s, shouldHashId, vocs);
if (Optional.ofNullable(oaf).map(List::isEmpty).orElse(false)) {
return s; return s;
} }
return null; return null;
@ -235,8 +236,7 @@ public class GenerateEntitiesApplication {
try { try {
return OBJECT_MAPPER.readValue(s, clazz); return OBJECT_MAPPER.readValue(s, clazz);
} catch (final Exception e) { } catch (final Exception e) {
log.error("Error parsing object of class: {}", clazz); log.error("Error parsing object of class: {}:\n{}", clazz, s);
log.error(s);
throw new IllegalArgumentException(e); throw new IllegalArgumentException(e);
} }
} }

7
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java
View File

@ -926,11 +926,12 @@ class MappersTest {
} }
@Test @Test
void testNotWellFormed() throws IOException, DocumentException { void testNotWellFormed() throws IOException {
final String xml = IOUtils final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_notwellformed.xml"))); .toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_notwellformed.xml")));
assertEquals(null, new OafToOafMapper(vocs, false, true).processMdRecord(xml)); final List<Oaf> actual = new OafToOafMapper(vocs, false, true).processMdRecord(xml);
assertNotNull(actual);
assertTrue(actual.isEmpty());
} }
private void assertValidId(final String id) { private void assertValidId(final String id) {