lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

Dedup aliases, created when a dedup in a previous build has been merged in a new dedup, need to be marked as "deletedbyinference", since they are "merged" in the new dedup

This commit is contained in:
Giambattista Bloisi 2024-02-08 15:12:16 +01:00
parent e6bdee86d1
commit b19643f6eb
1 changed files with 31 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/DedupRecordFactory.java
View File

@ -122,10 +122,20 @@ public class DedupRecordFactory {
} }
return Stream return Stream
.concat(Stream.of(agg.getDedupId()), agg.aliases.stream()) .concat(
.map(id -> { Stream
.of(agg.getDedupId())
.map(id -> createDedupOafEntity(id, agg.entity, dataInfo, ts)),
agg.aliases
.stream()
.map(id -> createMergedDedupAliasOafEntity(id, agg.entity, dataInfo, ts)))
.iterator();
}, beanEncoder);
}
private static OafEntity createDedupOafEntity(String id, OafEntity base, DataInfo dataInfo, long ts) {
try { try {
OafEntity res = (OafEntity) BeanUtils.cloneBean(agg.entity); OafEntity res = (OafEntity) BeanUtils.cloneBean(base);
res.setId(id); res.setId(id);
res.setDataInfo(dataInfo); res.setDataInfo(dataInfo);
res.setLastupdatetimestamp(ts); res.setLastupdatetimestamp(ts);
@ -133,9 +143,18 @@ public class DedupRecordFactory {
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
}) }
.iterator();
}, beanEncoder); private static OafEntity createMergedDedupAliasOafEntity(String id, OafEntity base, DataInfo dataInfo, long ts) {
try {
OafEntity res = createDedupOafEntity(id, base, dataInfo, ts);
DataInfo ds = (DataInfo) BeanUtils.cloneBean(dataInfo);
ds.setDeletedbyinference(true);
res.setDataInfo(ds);
return res;
} catch (Exception e) {
throw new RuntimeException(e);
}
} }
private static OafEntity reduceEntity(OafEntity entity, OafEntity duplicate) { private static OafEntity reduceEntity(OafEntity entity, OafEntity duplicate) {