spyros
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

filter out authors pid (only orcid). Added check to get unique provenance for context id. filtr out countries with code UNKNOWN

This commit is contained in:
Miriam Baglioni 2020-08-07 17:48:18 +02:00
parent 5ceb8c5f0a
commit 1b3ad1bce6
1 changed files with 119 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java
View File

@ -5,6 +5,8 @@ import java.io.Serializable;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.dump.oaf.*;
import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;
@ -125,10 +127,6 @@ public class ResultMapper implements Serializable {
break;
}
Optional
.ofNullable(input.getFulltext())
.ifPresent(fts -> out.setFulltext(fts.stream().map(ft -> ft.getValue()).collect(Collectors.toList())));
Optional
.ofNullable(input.getAuthor())
.ifPresent(ats -> out.setAuthor(ats.stream().map(at -> getAuthor(at)).collect(Collectors.toList())));
@ -155,33 +153,39 @@ public class ResultMapper implements Serializable {
.ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue())));
out.setContributor(contributorList);
List<Country> countryList = new ArrayList<>();
// List<Country> countryList = new ArrayList<>();
Optional
.ofNullable(input.getCountry())
.ifPresent(
value -> value
.stream()
.forEach(
c -> {
Country country = new Country();
country.setCode(c.getClassid());
country.setLabel(c.getClassname());
Optional
.ofNullable(c.getDataInfo())
.ifPresent(
provenance -> country
.setProvenance(
Provenance
.newInstance(
provenance
.getProvenanceaction()
.getClassname(),
c.getDataInfo().getTrust())));
countryList
.add(country);
}));
value -> out
.setCountry(
value
.stream()
.map(
c -> {
if (c.getClassid().equals((ModelConstants.UNKNOWN))) {
return null;
}
Country country = new Country();
country.setCode(c.getClassid());
country.setLabel(c.getClassname());
Optional
.ofNullable(c.getDataInfo())
.ifPresent(
provenance -> country
.setProvenance(
Provenance
.newInstance(
provenance
.getProvenanceaction()
.getClassid(),
c.getDataInfo().getTrust())));
return country;
})
.filter(Objects::nonNull)
.collect(Collectors.toList())));
out.setCountry(countryList);
// out.setCountry(countryList);
final List<String> coverageList = new ArrayList<>();
Optional
@ -380,7 +384,8 @@ public class ResultMapper implements Serializable {
.orElse(null))
.filter(Objects::nonNull)
.collect(Collectors.toSet()));
context.setProvenance(provenance);
context.setProvenance(getUniqueProvenance(provenance));
}
return context;
}
@ -406,6 +411,50 @@ public class ResultMapper implements Serializable {
}
private static List<Provenance> getUniqueProvenance(List<Provenance> provenance) {
Provenance iProv = new Provenance();
// iProv.setProvenance(Constants.INFERRED);
Provenance hProv = new Provenance();
// hProv.setProvenance(Constants.HARVESTED);
Provenance lProv = new Provenance();
for (Provenance p : provenance) {
switch (p.getProvenance()) {
case Constants.HARVESTED:
hProv = getHighestTrust(hProv, p);
break;
case Constants.INFERRED:
iProv = getHighestTrust(iProv, p);
// To be removed as soon as the new beta run has been done
// this fixex issue of not set trust during bulktagging
if (StringUtils.isEmpty(iProv.getTrust())) {
iProv.setTrust(Constants.DEFAULT_TRUST);
}
break;
case Constants.USER_CLAIM:
lProv = getHighestTrust(lProv, p);
break;
}
}
return Arrays
.asList(iProv, hProv, lProv)
.stream()
.filter(p -> !StringUtils.isEmpty(p.getProvenance()))
.collect(Collectors.toList());
}
private static Provenance getHighestTrust(Provenance hProv, Provenance p) {
if (StringUtils.isNoneEmpty(hProv.getTrust(), p.getTrust()))
return hProv.getTrust().compareTo(p.getTrust()) > 0 ? hProv : p;
return (StringUtils.isEmpty(p.getTrust()) && !StringUtils.isEmpty(hProv.getTrust())) ? hProv : p;
}
private static Subject getSubject(StructuredProperty s) {
Subject subject = new Subject();
subject.setSubject(ControlledField.newInstance(s.getQualifier().getClassid(), s.getValue()));
@ -435,21 +484,53 @@ public class ResultMapper implements Serializable {
a.setName(oa.getName());
a.setSurname(oa.getSurname());
a.setRank(oa.getRank());
Optional
.ofNullable(oa.getPid())
.ifPresent(
value -> a
.setPids(
value
.stream()
.map(p -> getPid(p))
.collect(Collectors.toList())));
Optional<List<StructuredProperty>> oPids = Optional
.ofNullable(oa.getPid());
if (oPids.isPresent()) {
Pid pid = getOrcid(oPids.get());
if (pid != null) {
a.setPid(pid);
}
}
return a;
}
private static Pid getOrcid(List<StructuredProperty> p) {
for (StructuredProperty pid : p) {
if (pid.getQualifier().getClassid().equals(Constants.ORCID)) {
Optional<DataInfo> di = Optional.ofNullable(pid.getDataInfo());
if (di.isPresent()) {
return Pid
.newInstance(
ControlledField
.newInstance(
pid.getQualifier().getClassid(),
pid.getValue()),
Provenance
.newInstance(
di.get().getProvenanceaction().getClassname(),
di.get().getTrust()));
} else {
return Pid
.newInstance(
ControlledField
.newInstance(
pid.getQualifier().getClassid(),
pid.getValue())
);
}
}
}
return null;
}
private static Pid getPid(StructuredProperty p) {
Pid pid = new Pid();
pid.setPid(ControlledField.newInstance(p.getQualifier().getClassid(), p.getValue()));
pid.setId(ControlledField.newInstance(p.getQualifier().getClassid(), p.getValue()));
Optional<DataInfo> di = Optional.ofNullable(p.getDataInfo());
Provenance provenance = new Provenance();
if (di.isPresent()) {