D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

code formatted

This commit is contained in:
sandro.labruzzo 2024-11-13 09:51:13 +01:00
parent b0478c380e
commit a1d5ad5c26
3 changed files with 228 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

423
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java
View File

@ -16,8 +16,6 @@ import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
import eu.dnetlib.dhp.schema.common.EntityType;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
@ -25,8 +23,10 @@ import org.apache.commons.lang3.tuple.Pair;
import com.github.sisyphsu.dateparser.DateParserUtils; import com.github.sisyphsu.dateparser.DateParserUtils;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
import eu.dnetlib.dhp.oa.merge.AuthorMerger; import eu.dnetlib.dhp.oa.merge.AuthorMerger;
import eu.dnetlib.dhp.schema.common.AccessRightComparator; import eu.dnetlib.dhp.schema.common.AccessRightComparator;
import eu.dnetlib.dhp.schema.common.EntityType;
import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.common.ModelSupport;
import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.*;
@ -46,7 +46,7 @@ public class MergeUtils {
} }
public static <T extends Oaf> T mergeGroup(Iterator<T> oafEntityIterator, public static <T extends Oaf> T mergeGroup(Iterator<T> oafEntityIterator,
boolean checkDelegateAuthority, VocabularyGroup vocs) { boolean checkDelegateAuthority, VocabularyGroup vocs) {
ArrayList<T> sortedEntities = new ArrayList<>(); ArrayList<T> sortedEntities = new ArrayList<>();
oafEntityIterator.forEachRemaining(sortedEntities::add); oafEntityIterator.forEachRemaining(sortedEntities::add);
@ -74,14 +74,15 @@ public class MergeUtils {
if (!vocs.vocabularyExists(ModelConstants.DNET_RESULT_TYPOLOGIES)) { if (!vocs.vocabularyExists(ModelConstants.DNET_RESULT_TYPOLOGIES)) {
return (T) mergedResult; return (T) mergedResult;
} else { } else {
final Qualifier expectedResultType = vocs.lookupTermBySynonym( final Qualifier expectedResultType = vocs
.lookupTermBySynonym(
ModelConstants.DNET_RESULT_TYPOLOGIES, ModelConstants.DNET_RESULT_TYPOLOGIES,
i.getInstancetype().getClassid()); i.getInstancetype().getClassid());
if (Objects.isNull(expectedResultType)) { if (Objects.isNull(expectedResultType)) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"instance type not bound to any result type in dnet:result_typologies: " + "instance type not bound to any result type in dnet:result_typologies: " +
i.getInstancetype().getClassid()); i.getInstancetype().getClassid());
} }
// there is a clash among the result types // there is a clash among the result types
@ -122,10 +123,10 @@ public class MergeUtils {
return mergeRelation((Relation) left, (Relation) right); return mergeRelation((Relation) left, (Relation) right);
} else { } else {
throw new RuntimeException( throw new RuntimeException(
String String
.format( .format(
"MERGE_FROM_AND_GET incompatible types: %s, %s", "MERGE_FROM_AND_GET incompatible types: %s, %s",
left.getClass().getCanonicalName(), right.getClass().getCanonicalName())); left.getClass().getCanonicalName(), right.getClass().getCanonicalName()));
} }
} }
@ -164,10 +165,10 @@ public class MergeUtils {
return mergeProject((Project) left, (Project) right); return mergeProject((Project) left, (Project) right);
} else { } else {
throw new RuntimeException( throw new RuntimeException(
String String
.format( .format(
"MERGE_FROM_AND_GET incompatible types: %s, %s", "MERGE_FROM_AND_GET incompatible types: %s, %s",
left.getClass().getCanonicalName(), right.getClass().getCanonicalName())); left.getClass().getCanonicalName(), right.getClass().getCanonicalName()));
} }
} }
@ -258,7 +259,7 @@ public class MergeUtils {
} }
private static <T, K> List<T> mergeLists(final List<T> left, final List<T> right, int trust, private static <T, K> List<T> mergeLists(final List<T> left, final List<T> right, int trust,
Function<T, K> keyExtractor, BinaryOperator<T> merger) { Function<T, K> keyExtractor, BinaryOperator<T> merger) {
if (left == null || left.isEmpty()) { if (left == null || left.isEmpty()) {
return right != null ? right : new ArrayList<>(); return right != null ? right : new ArrayList<>();
} else if (right == null || right.isEmpty()) { } else if (right == null || right.isEmpty()) {
@ -269,11 +270,11 @@ public class MergeUtils {
List<T> l = trust >= 0 ? right : left; List<T> l = trust >= 0 ? right : left;
return new ArrayList<>(Stream return new ArrayList<>(Stream
.concat(h.stream(), l.stream()) .concat(h.stream(), l.stream())
.filter(Objects::nonNull) .filter(Objects::nonNull)
.distinct() .distinct()
.collect(Collectors.toMap(keyExtractor, v -> v, merger, LinkedHashMap::new)) .collect(Collectors.toMap(keyExtractor, v -> v, merger, LinkedHashMap::new))
.values()); .values());
} }
private static <T, K> List<T> unionDistinctLists(final List<T> left, final List<T> right, int trust) { private static <T, K> List<T> unionDistinctLists(final List<T> left, final List<T> right, int trust) {
@ -287,10 +288,10 @@ public class MergeUtils {
List<T> l = trust >= 0 ? right : left; List<T> l = trust >= 0 ? right : left;
return Stream return Stream
.concat(h.stream(), l.stream()) .concat(h.stream(), l.stream())
.filter(Objects::nonNull) .filter(Objects::nonNull)
.distinct() .distinct()
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
private static List<String> unionDistinctListOfString(final List<String> l, final List<String> r) { private static List<String> unionDistinctListOfString(final List<String> l, final List<String> r) {
@ -301,10 +302,10 @@ public class MergeUtils {
} }
return Stream return Stream
.concat(l.stream(), r.stream()) .concat(l.stream(), r.stream())
.filter(StringUtils::isNotBlank) .filter(StringUtils::isNotBlank)
.distinct() .distinct()
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
// TODO review // TODO review
@ -330,7 +331,7 @@ public class MergeUtils {
} }
private static List<StructuredProperty> unionTitle(List<StructuredProperty> left, List<StructuredProperty> right, private static List<StructuredProperty> unionTitle(List<StructuredProperty> left, List<StructuredProperty> right,
int trust) { int trust) {
if (left == null) { if (left == null) {
return right; return right;
} else if (right == null) { } else if (right == null) {
@ -341,10 +342,10 @@ public class MergeUtils {
List<StructuredProperty> l = trust >= 0 ? right : left; List<StructuredProperty> l = trust >= 0 ? right : left;
return Stream return Stream
.concat(h.stream(), l.stream()) .concat(h.stream(), l.stream())
.filter(Objects::isNull) .filter(Objects::isNull)
.distinct() .distinct()
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
/** /**
@ -379,8 +380,8 @@ public class MergeUtils {
merged.setPid(mergeLists(merged.getPid(), enrich.getPid(), trust, MergeUtils::spKeyExtractor, (p1, p2) -> p1)); merged.setPid(mergeLists(merged.getPid(), enrich.getPid(), trust, MergeUtils::spKeyExtractor, (p1, p2) -> p1));
merged.setDateofcollection(LocalDateTime.now().toString()); merged.setDateofcollection(LocalDateTime.now().toString());
merged merged
.setDateoftransformation( .setDateoftransformation(
chooseString(merged.getDateoftransformation(), enrich.getDateoftransformation(), trust)); chooseString(merged.getDateoftransformation(), enrich.getDateoftransformation(), trust));
merged.setExtraInfo(unionDistinctLists(merged.getExtraInfo(), enrich.getExtraInfo(), trust)); merged.setExtraInfo(unionDistinctLists(merged.getExtraInfo(), enrich.getExtraInfo(), trust));
// When merging records OAI provenance becomes null // When merging records OAI provenance becomes null
merged.setOaiprovenance(null); merged.setOaiprovenance(null);
@ -397,7 +398,7 @@ public class MergeUtils {
checkArgument(Objects.equals(merge.getTarget(), enrich.getTarget()), "target ids must be equal"); checkArgument(Objects.equals(merge.getTarget(), enrich.getTarget()), "target ids must be equal");
checkArgument(Objects.equals(merge.getRelType(), enrich.getRelType()), "relType(s) must be equal"); checkArgument(Objects.equals(merge.getRelType(), enrich.getRelType()), "relType(s) must be equal");
checkArgument( checkArgument(
Objects.equals(merge.getSubRelType(), enrich.getSubRelType()), "subRelType(s) must be equal"); Objects.equals(merge.getSubRelType(), enrich.getSubRelType()), "subRelType(s) must be equal");
checkArgument(Objects.equals(merge.getRelClass(), enrich.getRelClass()), "relClass(es) must be equal"); checkArgument(Objects.equals(merge.getRelClass(), enrich.getRelClass()), "relClass(es) must be equal");
// merge.setProvenance(mergeLists(merge.getProvenance(), enrich.getProvenance())); // merge.setProvenance(mergeLists(merge.getProvenance(), enrich.getProvenance()));
@ -408,10 +409,10 @@ public class MergeUtils {
merge.setValidationDate(ModelSupport.oldest(merge.getValidationDate(), enrich.getValidationDate())); merge.setValidationDate(ModelSupport.oldest(merge.getValidationDate(), enrich.getValidationDate()));
} catch (ParseException e) { } catch (ParseException e) {
throw new IllegalArgumentException(String throw new IllegalArgumentException(String
.format( .format(
"invalid validation date format in relation [s:%s, t:%s]: %s", merge.getSource(), "invalid validation date format in relation [s:%s, t:%s]: %s", merge.getSource(),
merge.getTarget(), merge.getTarget(),
merge.getValidationDate())); merge.getValidationDate()));
} }
// TODO keyvalue merge // TODO keyvalue merge
@ -425,7 +426,7 @@ public class MergeUtils {
T merge = mergeOafEntityFields(original, enrich, trust); T merge = mergeOafEntityFields(original, enrich, trust);
if (merge.getProcessingchargeamount() == null if (merge.getProcessingchargeamount() == null
|| StringUtils.isBlank(merge.getProcessingchargeamount().getValue())) { || StringUtils.isBlank(merge.getProcessingchargeamount().getValue())) {
merge.setProcessingchargeamount(enrich.getProcessingchargeamount()); merge.setProcessingchargeamount(enrich.getProcessingchargeamount());
merge.setProcessingchargecurrency(enrich.getProcessingchargecurrency()); merge.setProcessingchargecurrency(enrich.getProcessingchargecurrency());
} }
@ -457,8 +458,8 @@ public class MergeUtils {
} }
merge merge
.setDateofacceptance( .setDateofacceptance(
mergeDateOfAcceptance(merge.getDateofacceptance(), enrich.getDateofacceptance(), trust)); mergeDateOfAcceptance(merge.getDateofacceptance(), enrich.getDateofacceptance(), trust));
merge.setPublisher(coalesce(merge.getPublisher(), enrich.getPublisher())); merge.setPublisher(coalesce(merge.getPublisher(), enrich.getPublisher()));
merge.setEmbargoenddate(coalesce(merge.getEmbargoenddate(), enrich.getEmbargoenddate())); merge.setEmbargoenddate(coalesce(merge.getEmbargoenddate(), enrich.getEmbargoenddate()));
@ -473,7 +474,7 @@ public class MergeUtils {
merge.setCoverage(unionDistinctLists(merge.getCoverage(), enrich.getCoverage(), trust)); merge.setCoverage(unionDistinctLists(merge.getCoverage(), enrich.getCoverage(), trust));
if (enrich.getBestaccessright() != null if (enrich.getBestaccessright() != null
&& new AccessRightComparator<>() && new AccessRightComparator<>()
.compare(enrich.getBestaccessright(), merge.getBestaccessright()) < 0) { .compare(enrich.getBestaccessright(), merge.getBestaccessright()) < 0) {
merge.setBestaccessright(enrich.getBestaccessright()); merge.setBestaccessright(enrich.getBestaccessright());
} }
@ -486,8 +487,8 @@ public class MergeUtils {
// ok // ok
merge merge
.setExternalReference( .setExternalReference(
mergeExternalReference(merge.getExternalReference(), enrich.getExternalReference(), trust)); mergeExternalReference(merge.getExternalReference(), enrich.getExternalReference(), trust));
// instance enrichment or union // instance enrichment or union
// review instance equals => add pid to comparision // review instance equals => add pid to comparision
@ -495,17 +496,17 @@ public class MergeUtils {
merge.setInstance(mergeInstances(merge.getInstance(), enrich.getInstance(), trust)); merge.setInstance(mergeInstances(merge.getInstance(), enrich.getInstance(), trust));
} else { } else {
final List<Instance> enrichmentInstances = isAnEnrichment(merge) ? merge.getInstance() final List<Instance> enrichmentInstances = isAnEnrichment(merge) ? merge.getInstance()
: enrich.getInstance(); : enrich.getInstance();
final List<Instance> enrichedInstances = isAnEnrichment(merge) ? enrich.getInstance() final List<Instance> enrichedInstances = isAnEnrichment(merge) ? enrich.getInstance()
: merge.getInstance(); : merge.getInstance();
if (isAnEnrichment(merge)) if (isAnEnrichment(merge))
merge.setDataInfo(enrich.getDataInfo()); merge.setDataInfo(enrich.getDataInfo());
merge.setInstance(enrichInstances(enrichedInstances, enrichmentInstances)); merge.setInstance(enrichInstances(enrichedInstances, enrichmentInstances));
} }
merge merge
.setEoscifguidelines( .setEoscifguidelines(
mergeEosciifguidelines(merge.getEoscifguidelines(), enrich.getEoscifguidelines(), trust)); mergeEosciifguidelines(merge.getEoscifguidelines(), enrich.getEoscifguidelines(), trust));
merge.setIsGreen(booleanOR(merge.getIsGreen(), enrich.getIsGreen())); merge.setIsGreen(booleanOR(merge.getIsGreen(), enrich.getIsGreen()));
// OK but should be list of values // OK but should be list of values
merge.setOpenAccessColor(coalesce(merge.getOpenAccessColor(), enrich.getOpenAccessColor())); merge.setOpenAccessColor(coalesce(merge.getOpenAccessColor(), enrich.getOpenAccessColor()));
@ -531,7 +532,7 @@ public class MergeUtils {
LocalDate enrich_date = LocalDate.parse(enrich.getValue(), DateTimeFormatter.ISO_DATE); LocalDate enrich_date = LocalDate.parse(enrich.getValue(), DateTimeFormatter.ISO_DATE);
if (enrich_date.getYear() > 1300 if (enrich_date.getYear() > 1300
&& (merge_date.getYear() < 1300 || merge_date.isAfter(enrich_date))) { && (merge_date.getYear() < 1300 || merge_date.isAfter(enrich_date))) {
return enrich; return enrich;
} }
} catch (NullPointerException | DateTimeParseException e) { } catch (NullPointerException | DateTimeParseException e) {
@ -549,56 +550,56 @@ public class MergeUtils {
private static List<Instance> mergeInstances(List<Instance> v1, List<Instance> v2, int trust) { private static List<Instance> mergeInstances(List<Instance> v1, List<Instance> v2, int trust) {
return mergeLists( return mergeLists(
v1, v2, trust, v1, v2, trust,
MergeUtils::instanceKeyExtractor, MergeUtils::instanceKeyExtractor,
MergeUtils::instanceMerger); MergeUtils::instanceMerger);
} }
private static List<EoscIfGuidelines> mergeEosciifguidelines(List<EoscIfGuidelines> v1, List<EoscIfGuidelines> v2, private static List<EoscIfGuidelines> mergeEosciifguidelines(List<EoscIfGuidelines> v1, List<EoscIfGuidelines> v2,
int trust) { int trust) {
return mergeLists( return mergeLists(
v1, v2, trust, er -> Joiner v1, v2, trust, er -> Joiner
.on("||") .on("||")
.useForNull("") .useForNull("")
.join(er.getCode(), er.getLabel(), er.getUrl(), er.getSemanticRelation()), .join(er.getCode(), er.getLabel(), er.getUrl(), er.getSemanticRelation()),
(r, l) -> r); (r, l) -> r);
} }
private static List<ExternalReference> mergeExternalReference(List<ExternalReference> v1, private static List<ExternalReference> mergeExternalReference(List<ExternalReference> v1,
List<ExternalReference> v2, int trust) { List<ExternalReference> v2, int trust) {
return mergeLists( return mergeLists(
v1, v2, trust, er -> Joiner v1, v2, trust, er -> Joiner
.on(',') .on(',')
.useForNull("") .useForNull("")
.join( .join(
er.getSitename(), er.getLabel(), er.getSitename(), er.getLabel(),
er.getUrl(), toString(er.getQualifier()), er.getRefidentifier(), er.getUrl(), toString(er.getQualifier()), er.getRefidentifier(),
er.getQuery(), toString(er.getDataInfo())), er.getQuery(), toString(er.getDataInfo())),
(r, l) -> r); (r, l) -> r);
} }
private static String toString(DataInfo di) { private static String toString(DataInfo di) {
return Joiner return Joiner
.on(',') .on(',')
.useForNull("") .useForNull("")
.join( .join(
di.getInvisible(), di.getInferred(), di.getDeletedbyinference(), di.getTrust(), di.getInvisible(), di.getInferred(), di.getDeletedbyinference(), di.getTrust(),
di.getInferenceprovenance(), toString(di.getProvenanceaction())); di.getInferenceprovenance(), toString(di.getProvenanceaction()));
} }
private static String toString(Qualifier q) { private static String toString(Qualifier q) {
return Joiner return Joiner
.on(',') .on(',')
.useForNull("") .useForNull("")
.join(q.getClassid(), q.getClassname(), q.getSchemeid(), q.getSchemename()); .join(q.getClassid(), q.getClassname(), q.getSchemeid(), q.getSchemename());
} }
private static String toString(StructuredProperty sp) { private static String toString(StructuredProperty sp) {
return Joiner return Joiner
.on(',') .on(',')
.useForNull("") .useForNull("")
.join(toString(sp.getQualifier()), sp.getValue()); .join(toString(sp.getQualifier()), sp.getValue());
} }
private static <T extends StructuredProperty> List<T> mergeStructuredProperties(List<T> v1, List<T> v2, int trust) { private static <T extends StructuredProperty> List<T> mergeStructuredProperties(List<T> v1, List<T> v2, int trust) {
@ -637,17 +638,17 @@ public class MergeUtils {
// 2. @@ // 2. @@
// 3. || // 3. ||
return String return String
.join( .join(
"::", "::",
kvKeyExtractor(i.getHostedby()), kvKeyExtractor(i.getHostedby()),
kvKeyExtractor(i.getCollectedfrom()), kvKeyExtractor(i.getCollectedfrom()),
qualifierKeyExtractor(i.getAccessright()), qualifierKeyExtractor(i.getAccessright()),
qualifierKeyExtractor(i.getInstancetype()), qualifierKeyExtractor(i.getInstancetype()),
Optional.ofNullable(i.getUrl()).map(u -> String.join("@@", u)).orElse(null), Optional.ofNullable(i.getUrl()).map(u -> String.join("@@", u)).orElse(null),
Optional Optional
.ofNullable(i.getPid()) .ofNullable(i.getPid())
.map(pp -> pp.stream().map(MergeUtils::spKeyExtractor).collect(Collectors.joining("@@"))) .map(pp -> pp.stream().map(MergeUtils::spKeyExtractor).collect(Collectors.joining("@@")))
.orElse(null)); .orElse(null));
} }
private static Instance instanceMerger(Instance i1, Instance i2) { private static Instance instanceMerger(Instance i1, Instance i2) {
@ -658,30 +659,30 @@ public class MergeUtils {
i.setInstancetype(i1.getInstancetype()); i.setInstancetype(i1.getInstancetype());
i.setPid(mergeLists(i1.getPid(), i2.getPid(), 0, MergeUtils::spKeyExtractor, (sp1, sp2) -> sp1)); i.setPid(mergeLists(i1.getPid(), i2.getPid(), 0, MergeUtils::spKeyExtractor, (sp1, sp2) -> sp1));
i i
.setAlternateIdentifier( .setAlternateIdentifier(
mergeLists( mergeLists(
i1.getAlternateIdentifier(), i2.getAlternateIdentifier(), 0, MergeUtils::spKeyExtractor, i1.getAlternateIdentifier(), i2.getAlternateIdentifier(), 0, MergeUtils::spKeyExtractor,
(sp1, sp2) -> sp1)); (sp1, sp2) -> sp1));
i i
.setRefereed( .setRefereed(
Collections Collections
.min( .min(
Stream.of(i1.getRefereed(), i2.getRefereed()).collect(Collectors.toList()), Stream.of(i1.getRefereed(), i2.getRefereed()).collect(Collectors.toList()),
new RefereedComparator())); new RefereedComparator()));
i i
.setInstanceTypeMapping( .setInstanceTypeMapping(
mergeLists( mergeLists(
i1.getInstanceTypeMapping(), i2.getInstanceTypeMapping(), 0, i1.getInstanceTypeMapping(), i2.getInstanceTypeMapping(), 0,
MergeUtils::instanceTypeMappingKeyExtractor, (itm1, itm2) -> itm1)); MergeUtils::instanceTypeMappingKeyExtractor, (itm1, itm2) -> itm1));
i.setFulltext(selectFulltext(i1.getFulltext(), i2.getFulltext())); i.setFulltext(selectFulltext(i1.getFulltext(), i2.getFulltext()));
i.setDateofacceptance(selectOldestDate(i1.getDateofacceptance(), i2.getDateofacceptance())); i.setDateofacceptance(selectOldestDate(i1.getDateofacceptance(), i2.getDateofacceptance()));
i.setLicense(coalesce(i1.getLicense(), i2.getLicense())); i.setLicense(coalesce(i1.getLicense(), i2.getLicense()));
i.setProcessingchargeamount(coalesce(i1.getProcessingchargeamount(), i2.getProcessingchargeamount())); i.setProcessingchargeamount(coalesce(i1.getProcessingchargeamount(), i2.getProcessingchargeamount()));
i.setProcessingchargecurrency(coalesce(i1.getProcessingchargecurrency(), i2.getProcessingchargecurrency())); i.setProcessingchargecurrency(coalesce(i1.getProcessingchargecurrency(), i2.getProcessingchargecurrency()));
i i
.setMeasures( .setMeasures(
mergeLists(i1.getMeasures(), i2.getMeasures(), 0, MergeUtils::measureKeyExtractor, (m1, m2) -> m1)); mergeLists(i1.getMeasures(), i2.getMeasures(), 0, MergeUtils::measureKeyExtractor, (m1, m2) -> m1));
i.setUrl(unionDistinctListOfString(i1.getUrl(), i2.getUrl())); i.setUrl(unionDistinctListOfString(i1.getUrl(), i2.getUrl()));
@ -690,14 +691,14 @@ public class MergeUtils {
private static String measureKeyExtractor(Measure m) { private static String measureKeyExtractor(Measure m) {
return String return String
.join( .join(
"::", "::",
m.getId(), m.getId(),
m m
.getUnit() .getUnit()
.stream() .stream()
.map(KeyValue::getKey) .map(KeyValue::getKey)
.collect(Collectors.joining("::"))); .collect(Collectors.joining("::")));
} }
private static Field<String> selectOldestDate(Field<String> d1, Field<String> d2) { private static Field<String> selectOldestDate(Field<String> d1, Field<String> d2) {
@ -708,16 +709,16 @@ public class MergeUtils {
} }
return Stream return Stream
.of(d1, d2) .of(d1, d2)
.min( .min(
Comparator Comparator
.comparing( .comparing(
f -> DateParserUtils f -> DateParserUtils
.parseDate(f.getValue()) .parseDate(f.getValue())
.toInstant() .toInstant()
.atZone(ZoneId.systemDefault()) .atZone(ZoneId.systemDefault())
.toLocalDate())) .toLocalDate()))
.orElse(d1); .orElse(d1);
} }
private static String selectFulltext(String ft1, String ft2) { private static String selectFulltext(String ft1, String ft2) {
@ -732,12 +733,12 @@ public class MergeUtils {
private static String instanceTypeMappingKeyExtractor(InstanceTypeMapping itm) { private static String instanceTypeMappingKeyExtractor(InstanceTypeMapping itm) {
return String return String
.join( .join(
"::", "::",
itm.getOriginalType(), itm.getOriginalType(),
itm.getTypeCode(), itm.getTypeCode(),
itm.getTypeLabel(), itm.getTypeLabel(),
itm.getVocabularyName()); itm.getVocabularyName());
} }
private static String kvKeyExtractor(KeyValue kv) { private static String kvKeyExtractor(KeyValue kv) {
@ -754,13 +755,13 @@ public class MergeUtils {
private static String spKeyExtractor(StructuredProperty sp) { private static String spKeyExtractor(StructuredProperty sp) {
return Optional return Optional
.ofNullable(sp) .ofNullable(sp)
.map( .map(
s -> Joiner s -> Joiner
.on("||") .on("||")
.useForNull("") .useForNull("")
.join(qualifierKeyExtractor(s.getQualifier()), s.getValue())) .join(qualifierKeyExtractor(s.getQualifier()), s.getValue()))
.orElse(null); .orElse(null);
} }
private static <T extends OtherResearchProduct> T mergeORP(T original, T enrich) { private static <T extends OtherResearchProduct> T mergeORP(T original, T enrich) {
@ -782,8 +783,8 @@ public class MergeUtils {
merge.setLicense(unionDistinctLists(merge.getLicense(), enrich.getLicense(), trust)); merge.setLicense(unionDistinctLists(merge.getLicense(), enrich.getLicense(), trust));
merge.setCodeRepositoryUrl(chooseReference(merge.getCodeRepositoryUrl(), enrich.getCodeRepositoryUrl(), trust)); merge.setCodeRepositoryUrl(chooseReference(merge.getCodeRepositoryUrl(), enrich.getCodeRepositoryUrl(), trust));
merge merge
.setProgrammingLanguage( .setProgrammingLanguage(
chooseReference(merge.getProgrammingLanguage(), enrich.getProgrammingLanguage(), trust)); chooseReference(merge.getProgrammingLanguage(), enrich.getProgrammingLanguage(), trust));
return merge; return merge;
} }
@ -797,11 +798,11 @@ public class MergeUtils {
merge.setSize(chooseReference(merge.getSize(), enrich.getSize(), trust)); merge.setSize(chooseReference(merge.getSize(), enrich.getSize(), trust));
merge.setVersion(chooseReference(merge.getVersion(), enrich.getVersion(), trust)); merge.setVersion(chooseReference(merge.getVersion(), enrich.getVersion(), trust));
merge merge
.setLastmetadataupdate( .setLastmetadataupdate(
chooseReference(merge.getLastmetadataupdate(), enrich.getLastmetadataupdate(), trust)); chooseReference(merge.getLastmetadataupdate(), enrich.getLastmetadataupdate(), trust));
merge merge
.setMetadataversionnumber( .setMetadataversionnumber(
chooseReference(merge.getMetadataversionnumber(), enrich.getMetadataversionnumber(), trust)); chooseReference(merge.getMetadataversionnumber(), enrich.getMetadataversionnumber(), trust));
merge.setGeolocation(unionDistinctLists(merge.getGeolocation(), enrich.getGeolocation(), trust)); merge.setGeolocation(unionDistinctLists(merge.getGeolocation(), enrich.getGeolocation(), trust));
return merge; return merge;
@ -823,26 +824,26 @@ public class MergeUtils {
merged.setLegalshortname(chooseReference(merged.getLegalshortname(), enrich.getLegalshortname(), trust)); merged.setLegalshortname(chooseReference(merged.getLegalshortname(), enrich.getLegalshortname(), trust));
merged.setLegalname(chooseReference(merged.getLegalname(), enrich.getLegalname(), trust)); merged.setLegalname(chooseReference(merged.getLegalname(), enrich.getLegalname(), trust));
merged merged
.setAlternativeNames(unionDistinctLists(enrich.getAlternativeNames(), merged.getAlternativeNames(), trust)); .setAlternativeNames(unionDistinctLists(enrich.getAlternativeNames(), merged.getAlternativeNames(), trust));
merged.setWebsiteurl(chooseReference(merged.getWebsiteurl(), enrich.getWebsiteurl(), trust)); merged.setWebsiteurl(chooseReference(merged.getWebsiteurl(), enrich.getWebsiteurl(), trust));
merged.setLogourl(chooseReference(merged.getLogourl(), enrich.getLogourl(), trust)); merged.setLogourl(chooseReference(merged.getLogourl(), enrich.getLogourl(), trust));
merged.setEclegalbody(chooseReference(merged.getEclegalbody(), enrich.getEclegalbody(), trust)); merged.setEclegalbody(chooseReference(merged.getEclegalbody(), enrich.getEclegalbody(), trust));
merged.setEclegalperson(chooseReference(merged.getEclegalperson(), enrich.getEclegalperson(), trust)); merged.setEclegalperson(chooseReference(merged.getEclegalperson(), enrich.getEclegalperson(), trust));
merged.setEcnonprofit(chooseReference(merged.getEcnonprofit(), enrich.getEcnonprofit(), trust)); merged.setEcnonprofit(chooseReference(merged.getEcnonprofit(), enrich.getEcnonprofit(), trust));
merged merged
.setEcresearchorganization( .setEcresearchorganization(
chooseReference(merged.getEcresearchorganization(), enrich.getEcresearchorganization(), trust)); chooseReference(merged.getEcresearchorganization(), enrich.getEcresearchorganization(), trust));
merged merged
.setEchighereducation(chooseReference(merged.getEchighereducation(), enrich.getEchighereducation(), trust)); .setEchighereducation(chooseReference(merged.getEchighereducation(), enrich.getEchighereducation(), trust));
merged merged
.setEcinternationalorganizationeurinterests( .setEcinternationalorganizationeurinterests(
chooseReference( chooseReference(
merged.getEcinternationalorganizationeurinterests(), merged.getEcinternationalorganizationeurinterests(),
enrich.getEcinternationalorganizationeurinterests(), trust)); enrich.getEcinternationalorganizationeurinterests(), trust));
merged merged
.setEcinternationalorganization( .setEcinternationalorganization(
chooseReference( chooseReference(
merged.getEcinternationalorganization(), enrich.getEcinternationalorganization(), trust)); merged.getEcinternationalorganization(), enrich.getEcinternationalorganization(), trust));
merged.setEcenterprise(chooseReference(merged.getEcenterprise(), enrich.getEcenterprise(), trust)); merged.setEcenterprise(chooseReference(merged.getEcenterprise(), enrich.getEcenterprise(), trust));
merged.setEcsmevalidated(chooseReference(merged.getEcsmevalidated(), enrich.getEcsmevalidated(), trust)); merged.setEcsmevalidated(chooseReference(merged.getEcsmevalidated(), enrich.getEcsmevalidated(), trust));
merged.setEcnutscode(chooseReference(merged.getEcnutscode(), enrich.getEcnutscode(), trust)); merged.setEcnutscode(chooseReference(merged.getEcnutscode(), enrich.getEcnutscode(), trust));
@ -866,8 +867,8 @@ public class MergeUtils {
merged.setDuration(chooseReference(merged.getDuration(), enrich.getDuration(), trust)); merged.setDuration(chooseReference(merged.getDuration(), enrich.getDuration(), trust));
merged.setEcsc39(chooseReference(merged.getEcsc39(), enrich.getEcsc39(), trust)); merged.setEcsc39(chooseReference(merged.getEcsc39(), enrich.getEcsc39(), trust));
merged merged
.setOamandatepublications( .setOamandatepublications(
chooseReference(merged.getOamandatepublications(), enrich.getOamandatepublications(), trust)); chooseReference(merged.getOamandatepublications(), enrich.getOamandatepublications(), trust));
merged.setEcarticle29_3(chooseReference(merged.getEcarticle29_3(), enrich.getEcarticle29_3(), trust)); merged.setEcarticle29_3(chooseReference(merged.getEcarticle29_3(), enrich.getEcarticle29_3(), trust));
merged.setSubjects(unionDistinctLists(merged.getSubjects(), enrich.getSubjects(), trust)); merged.setSubjects(unionDistinctLists(merged.getSubjects(), enrich.getSubjects(), trust));
merged.setFundingtree(unionDistinctLists(merged.getFundingtree(), enrich.getFundingtree(), trust)); merged.setFundingtree(unionDistinctLists(merged.getFundingtree(), enrich.getFundingtree(), trust));
@ -893,8 +894,8 @@ public class MergeUtils {
} }
merged merged
.setH2020classification( .setH2020classification(
unionDistinctLists(merged.getH2020classification(), enrich.getH2020classification(), trust)); unionDistinctLists(merged.getH2020classification(), enrich.getH2020classification(), trust));
return merged; return merged;
} }
@ -921,7 +922,7 @@ public class MergeUtils {
* @return list of instances possibly enriched * @return list of instances possibly enriched
*/ */
private static List<Instance> enrichInstances(final List<Instance> toEnrichInstances, private static List<Instance> enrichInstances(final List<Instance> toEnrichInstances,
final List<Instance> enrichmentInstances) { final List<Instance> enrichmentInstances) {
final List<Instance> enrichmentResult = new ArrayList<>(); final List<Instance> enrichmentResult = new ArrayList<>();
if (toEnrichInstances == null) { if (toEnrichInstances == null) {
@ -959,42 +960,42 @@ public class MergeUtils {
*/ */
private static Map<String, Instance> toInstanceMap(final List<Instance> ri) { private static Map<String, Instance> toInstanceMap(final List<Instance> ri) {
return ri return ri
.stream() .stream()
.filter(i -> i.getPid() != null || i.getAlternateIdentifier() != null) .filter(i -> i.getPid() != null || i.getAlternateIdentifier() != null)
.flatMap(i -> { .flatMap(i -> {
final List<Pair<String, Instance>> result = new ArrayList<>(); final List<Pair<String, Instance>> result = new ArrayList<>();
if (i.getPid() != null) if (i.getPid() != null)
i i
.getPid() .getPid()
.stream() .stream()
.filter(MergeUtils::validPid) .filter(MergeUtils::validPid)
.forEach(p -> result.add(new ImmutablePair<>(extractKeyFromPid(p), i))); .forEach(p -> result.add(new ImmutablePair<>(extractKeyFromPid(p), i)));
if (i.getAlternateIdentifier() != null) if (i.getAlternateIdentifier() != null)
i i
.getAlternateIdentifier() .getAlternateIdentifier()
.stream() .stream()
.filter(MergeUtils::validPid) .filter(MergeUtils::validPid)
.forEach(p -> result.add(new ImmutablePair<>(extractKeyFromPid(p), i))); .forEach(p -> result.add(new ImmutablePair<>(extractKeyFromPid(p), i)));
return result.stream(); return result.stream();
}) })
.collect( .collect(
Collectors Collectors
.toMap( .toMap(
Pair::getLeft, Pair::getLeft,
Pair::getRight, Pair::getRight,
(a, b) -> a)); (a, b) -> a));
} }
private static boolean isFromDelegatedAuthority(Result r) { private static boolean isFromDelegatedAuthority(Result r) {
return Optional return Optional
.ofNullable(r.getInstance()) .ofNullable(r.getInstance())
.map( .map(
instance -> instance instance -> instance
.stream() .stream()
.filter(i -> Objects.nonNull(i.getCollectedfrom())) .filter(i -> Objects.nonNull(i.getCollectedfrom()))
.map(i -> i.getCollectedfrom().getKey()) .map(i -> i.getCollectedfrom().getKey())
.anyMatch(cfId -> IdentifierFactory.delegatedAuthorityDatasourceIds().contains(cfId))) .anyMatch(cfId -> IdentifierFactory.delegatedAuthorityDatasourceIds().contains(cfId)))
.orElse(false); .orElse(false);
} }
/** /**
@ -1030,15 +1031,15 @@ public class MergeUtils {
* @return the list * @return the list
*/ */
private static List<Instance> findEnrichmentsByPID(final List<StructuredProperty> pids, private static List<Instance> findEnrichmentsByPID(final List<StructuredProperty> pids,
final Map<String, Instance> enrichments) { final Map<String, Instance> enrichments) {
if (pids == null || enrichments == null) if (pids == null || enrichments == null)
return null; return null;
return pids return pids
.stream() .stream()
.map(MergeUtils::extractKeyFromPid) .map(MergeUtils::extractKeyFromPid)
.map(enrichments::get) .map(enrichments::get)
.filter(Objects::nonNull) .filter(Objects::nonNull)
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
/** /**
@ -1049,8 +1050,8 @@ public class MergeUtils {
*/ */
private static boolean isAnEnrichment(OafEntity e) { private static boolean isAnEnrichment(OafEntity e) {
return e.getDataInfo() != null && return e.getDataInfo() != null &&
e.getDataInfo().getProvenanceaction() != null e.getDataInfo().getProvenanceaction() != null
&& ModelConstants.PROVENANCE_ENRICH.equalsIgnoreCase(e.getDataInfo().getProvenanceaction().getClassid()); && ModelConstants.PROVENANCE_ENRICH.equalsIgnoreCase(e.getDataInfo().getProvenanceaction().getClassid());
} }
/** /**
@ -1073,17 +1074,17 @@ public class MergeUtils {
merge.setHostedby(firstNonNull(merge.getHostedby(), enrichment.getHostedby())); merge.setHostedby(firstNonNull(merge.getHostedby(), enrichment.getHostedby()));
merge.setUrl(unionDistinctLists(merge.getUrl(), enrichment.getUrl(), 0)); merge.setUrl(unionDistinctLists(merge.getUrl(), enrichment.getUrl(), 0));
merge merge
.setDistributionlocation( .setDistributionlocation(
firstNonNull(merge.getDistributionlocation(), enrichment.getDistributionlocation())); firstNonNull(merge.getDistributionlocation(), enrichment.getDistributionlocation()));
merge.setCollectedfrom(firstNonNull(merge.getCollectedfrom(), enrichment.getCollectedfrom())); merge.setCollectedfrom(firstNonNull(merge.getCollectedfrom(), enrichment.getCollectedfrom()));
// pid and alternateId are used for matching // pid and alternateId are used for matching
merge.setDateofacceptance(firstNonNull(merge.getDateofacceptance(), enrichment.getDateofacceptance())); merge.setDateofacceptance(firstNonNull(merge.getDateofacceptance(), enrichment.getDateofacceptance()));
merge merge
.setProcessingchargeamount( .setProcessingchargeamount(
firstNonNull(merge.getProcessingchargeamount(), enrichment.getProcessingchargeamount())); firstNonNull(merge.getProcessingchargeamount(), enrichment.getProcessingchargeamount()));
merge merge
.setProcessingchargecurrency( .setProcessingchargecurrency(
firstNonNull(merge.getProcessingchargecurrency(), enrichment.getProcessingchargecurrency())); firstNonNull(merge.getProcessingchargecurrency(), enrichment.getProcessingchargecurrency()));
merge.setRefereed(firstNonNull(merge.getRefereed(), enrichment.getRefereed())); merge.setRefereed(firstNonNull(merge.getRefereed(), enrichment.getRefereed()));
merge.setMeasures(unionDistinctLists(merge.getMeasures(), enrichment.getMeasures(), 0)); merge.setMeasures(unionDistinctLists(merge.getMeasures(), enrichment.getMeasures(), 0));
merge.setFulltext(firstNonNull(merge.getFulltext(), enrichment.getFulltext())); merge.setFulltext(firstNonNull(merge.getFulltext(), enrichment.getFulltext()));
@ -1091,14 +1092,14 @@ public class MergeUtils {
private static int compareTrust(Oaf a, Oaf b) { private static int compareTrust(Oaf a, Oaf b) {
String left = Optional String left = Optional
.ofNullable(a.getDataInfo()) .ofNullable(a.getDataInfo())
.map(DataInfo::getTrust) .map(DataInfo::getTrust)
.orElse("0.0"); .orElse("0.0");
String right = Optional String right = Optional
.ofNullable(b.getDataInfo()) .ofNullable(b.getDataInfo())
.map(DataInfo::getTrust) .map(DataInfo::getTrust)
.orElse("0.0"); .orElse("0.0");
return left.compareTo(right); return left.compareTo(right);
} }

22
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/personentity/ExtractPerson.java
View File

@ -346,15 +346,19 @@ public class ExtractPerson implements Serializable {
.structuredProperty( .structuredProperty(
op.getOrcid(), ModelConstants.ORCID, ModelConstants.ORCID_CLASSNAME, op.getOrcid(), ModelConstants.ORCID, ModelConstants.ORCID_CLASSNAME,
ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES,
OafMapperUtils.dataInfo(false, OafMapperUtils
null, .dataInfo(
false, false,
false, null,
OafMapperUtils.qualifier(ModelConstants.SYSIMPORT_CROSSWALK_ENTITYREGISTRY, false,
ModelConstants.SYSIMPORT_CROSSWALK_ENTITYREGISTRY, false,
ModelConstants.DNET_PID_TYPES, OafMapperUtils
ModelConstants.DNET_PID_TYPES), .qualifier(
"0.91"))); ModelConstants.SYSIMPORT_CROSSWALK_ENTITYREGISTRY,
ModelConstants.SYSIMPORT_CROSSWALK_ENTITYREGISTRY,
ModelConstants.DNET_PID_TYPES,
ModelConstants.DNET_PID_TYPES),
"0.91")));
person.setDateofcollection(op.getLastModifiedDate()); person.setDateofcollection(op.getLastModifiedDate());
person.setOriginalId(Arrays.asList(op.getOrcid())); person.setOriginalId(Arrays.asList(op.getOrcid()));
person.setDataInfo(ORCIDDATAINFO); person.setDataInfo(ORCIDDATAINFO);

6
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIterator.java
View File

@ -3,6 +3,7 @@ package eu.dnetlib.dhp.collection.plugin.gtr2;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
@ -18,7 +19,6 @@ import org.dom4j.Document;
import org.dom4j.DocumentException; import org.dom4j.DocumentException;
import org.dom4j.DocumentHelper; import org.dom4j.DocumentHelper;
import org.dom4j.Element; import org.dom4j.Element;
import java.time.format.DateTimeFormatter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -188,11 +188,11 @@ public class Gtr2PublicationsIterator implements Iterator<String> {
private Document loadURL(final String cleanUrl, final int attempt) { private Document loadURL(final String cleanUrl, final int attempt) {
try { try {
log.debug(" * Downloading Url: {}", cleanUrl); log.debug(" * Downloading Url: {}", cleanUrl);
final byte[] bytes = this.connector.getInputSource(cleanUrl).getBytes(StandardCharsets.UTF_8); final byte[] bytes = this.connector.getInputSource(cleanUrl).getBytes(StandardCharsets.UTF_8);
return DocumentHelper.parseText(new String(bytes)); return DocumentHelper.parseText(new String(bytes));
} catch (final Throwable e) { } catch (final Throwable e) {
log.error("Error dowloading url: {}, attempt = {}", cleanUrl, attempt, e); log.error("Error dowloading url: {}, attempt = {}", cleanUrl, attempt, e);
if (attempt >= MAX_ATTEMPTS) { if (attempt >= MAX_ATTEMPTS) {
throw new RuntimeException("Error downloading url: " + cleanUrl, e); throw new RuntimeException("Error downloading url: " + cleanUrl, e);
} }