dimitris.pierrakos
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

[aggregator graph] save invalid records aside for further inspection

This commit is contained in:
Claudio Atzori 2022-09-15 10:49:42 +02:00
parent 9e7ec4198f
commit 1e42d984e1
3 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java
View File

@ -143,7 +143,7 @@ public abstract class AbstractMdRecordToOafMapper {
return createOafs(doc, type, instances, collectedFrom, info, lastUpdateTimestamp);
} catch (DocumentException e) {
log.error("Error with record:\n" + xml);
return null;
return Lists.newArrayList();
}
}

10
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java
View File

@ -150,8 +150,8 @@ public class GenerateEntitiesApplication {
.sequenceFile(sp, Text.class, Text.class)
.map(k -> new Tuple2<>(k._1().toString(), k._2().toString()))
.map(k -> convertToListOaf(k._1(), k._2(), shouldHashId, vocs))
.filter(Objects::nonNull)
.flatMap(List::iterator));
.flatMap(List::iterator)
.filter(Objects::nonNull));
}
switch (mode) {
@ -225,7 +225,8 @@ public class GenerateEntitiesApplication {
final boolean shouldHashId,
final VocabularyGroup vocs) {
if (Objects.isNull(convertToListOaf(id, s, shouldHashId, vocs))) {
final List<Oaf> oaf = convertToListOaf(id, s, shouldHashId, vocs);
if (Optional.ofNullable(oaf).map(List::isEmpty).orElse(false)) {
return s;
}
return null;
@ -235,8 +236,7 @@ public class GenerateEntitiesApplication {
try {
return OBJECT_MAPPER.readValue(s, clazz);
} catch (final Exception e) {
log.error("Error parsing object of class: {}", clazz);
log.error(s);
log.error("Error parsing object of class: {}:\n{}", clazz, s);
throw new IllegalArgumentException(e);
}
}

7
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java
View File

@ -926,11 +926,12 @@ class MappersTest {
}
@Test
void testNotWellFormed() throws IOException, DocumentException {
void testNotWellFormed() throws IOException {
final String xml = IOUtils
.toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_notwellformed.xml")));
assertEquals(null, new OafToOafMapper(vocs, false, true).processMdRecord(xml));
final List<Oaf> actual = new OafToOafMapper(vocs, false, true).processMdRecord(xml);
assertNotNull(actual);
assertTrue(actual.isEmpty());
}
private void assertValidId(final String id) {