sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

making XmlRecordFactory immune to graph encoding changes (mostly to avoid NPEs)

This commit is contained in:
Claudio Atzori 2020-05-22 08:50:44 +02:00
parent b33dd58be4
commit 925d933204
2 changed files with 62 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateFactory.java
View File

@ -5,6 +5,7 @@ import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.removePrefix;
import static eu.dnetlib.dhp.oa.provision.utils.XmlSerializationUtils.escapeXml; import static eu.dnetlib.dhp.oa.provision.utils.XmlSerializationUtils.escapeXml;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -95,7 +96,7 @@ public class TemplateFactory {
.add("metadata", instancemetadata) .add("metadata", instancemetadata)
.add( .add(
"webresources", "webresources",
webresources (webresources != null ? webresources : new ArrayList<String>())
.stream() .stream()
.filter(StringUtils::isNotBlank) .filter(StringUtils::isNotBlank)
.map(w -> getWebResource(w)) .map(w -> getWebResource(w))

77
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java
View File

@ -174,6 +174,7 @@ public class XmlRecordFactory implements Serializable {
entity entity
.getCollectedfrom() .getCollectedfrom()
.stream() .stream()
.filter(XmlRecordFactory::kvNotBlank)
.map(kv -> XmlSerializationUtils.mapKeyValue("collectedfrom", kv)) .map(kv -> XmlSerializationUtils.mapKeyValue("collectedfrom", kv))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -183,6 +184,7 @@ public class XmlRecordFactory implements Serializable {
entity entity
.getOriginalId() .getOriginalId()
.stream() .stream()
.filter(Objects::nonNull)
.map(s -> XmlSerializationUtils.asXmlElement("originalId", s)) .map(s -> XmlSerializationUtils.asXmlElement("originalId", s))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -192,6 +194,7 @@ public class XmlRecordFactory implements Serializable {
entity entity
.getPid() .getPid()
.stream() .stream()
.filter(Objects::nonNull)
.map(p -> XmlSerializationUtils.mapStructuredProperty("pid", p)) .map(p -> XmlSerializationUtils.mapStructuredProperty("pid", p))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -213,6 +216,7 @@ public class XmlRecordFactory implements Serializable {
r r
.getTitle() .getTitle()
.stream() .stream()
.filter(Objects::nonNull)
.map(t -> XmlSerializationUtils.mapStructuredProperty("title", t)) .map(t -> XmlSerializationUtils.mapStructuredProperty("title", t))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -225,6 +229,7 @@ public class XmlRecordFactory implements Serializable {
r r
.getAuthor() .getAuthor()
.stream() .stream()
.filter(Objects::nonNull)
.map( .map(
a -> { a -> {
final StringBuilder sb = new StringBuilder("<creator rank=\"" + a.getRank() + "\""); final StringBuilder sb = new StringBuilder("<creator rank=\"" + a.getRank() + "\"");
@ -240,24 +245,26 @@ public class XmlRecordFactory implements Serializable {
a a
.getPid() .getPid()
.stream() .stream()
.filter(Objects::nonNull)
.filter( .filter(
sp -> isNotBlank(sp.getQualifier().getClassid()) sp -> isNotBlank(sp.getQualifier().getClassid())
&& isNotBlank(sp.getValue())) && isNotBlank(sp.getValue()))
.collect(
Collectors
.toMap(
p -> getAuthorPidType(p.getQualifier().getClassid()),
p -> p,
(p1, p2) -> p1))
.values()
.forEach( .forEach(
sp -> { sp -> {
String pidType = XmlSerializationUtils String pidType = getAuthorPidType(sp.getQualifier().getClassid());
.escapeXml(
sp.getQualifier().getClassid())
.replaceAll("\\W", "");
String pidValue = XmlSerializationUtils.escapeXml(sp.getValue()); String pidValue = XmlSerializationUtils.escapeXml(sp.getValue());
// ugly hack: some records // ugly hack: some records provide swapped pidtype and pidvalue
// provide swapped pidtype and
// pidvalue
if (authorPidTypes.contains(pidValue.toLowerCase().trim())) { if (authorPidTypes.contains(pidValue.toLowerCase().trim())) {
sb.append(String.format(" %s=\"%s\"", pidValue, pidType)); sb.append(String.format(" %s=\"%s\"", pidValue, pidType));
} else { } else {
pidType = pidType.replaceAll("\\W", "").replaceAll("\\d", "");
if (isNotBlank(pidType)) { if (isNotBlank(pidType)) {
sb sb
.append( .append(
@ -285,6 +292,7 @@ public class XmlRecordFactory implements Serializable {
r r
.getContributor() .getContributor()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("contributor", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("contributor", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -294,6 +302,7 @@ public class XmlRecordFactory implements Serializable {
r r
.getCountry() .getCountry()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.mapQualifier("country", c)) .map(c -> XmlSerializationUtils.mapQualifier("country", c))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -303,6 +312,7 @@ public class XmlRecordFactory implements Serializable {
r r
.getCoverage() .getCoverage()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("coverage", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("coverage", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -319,6 +329,7 @@ public class XmlRecordFactory implements Serializable {
r r
.getDescription() .getDescription()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("description", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("description", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -333,6 +344,7 @@ public class XmlRecordFactory implements Serializable {
r r
.getSubject() .getSubject()
.stream() .stream()
.filter(Objects::nonNull)
.map(s -> XmlSerializationUtils.mapStructuredProperty("subject", s)) .map(s -> XmlSerializationUtils.mapStructuredProperty("subject", s))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -345,6 +357,7 @@ public class XmlRecordFactory implements Serializable {
r r
.getRelevantdate() .getRelevantdate()
.stream() .stream()
.filter(Objects::nonNull)
.map(s -> XmlSerializationUtils.mapStructuredProperty("relevantdate", s)) .map(s -> XmlSerializationUtils.mapStructuredProperty("relevantdate", s))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -357,6 +370,7 @@ public class XmlRecordFactory implements Serializable {
r r
.getSource() .getSource()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("source", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("source", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -366,6 +380,7 @@ public class XmlRecordFactory implements Serializable {
r r
.getFormat() .getFormat()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("format", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("format", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -429,6 +444,7 @@ public class XmlRecordFactory implements Serializable {
orp orp
.getContactperson() .getContactperson()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("contactperson", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("contactperson", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -439,6 +455,7 @@ public class XmlRecordFactory implements Serializable {
orp orp
.getContactgroup() .getContactgroup()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("contactgroup", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("contactgroup", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -448,6 +465,7 @@ public class XmlRecordFactory implements Serializable {
orp orp
.getTool() .getTool()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("tool", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("tool", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -461,6 +479,7 @@ public class XmlRecordFactory implements Serializable {
s s
.getDocumentationUrl() .getDocumentationUrl()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("documentationUrl", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("documentationUrl", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -470,6 +489,7 @@ public class XmlRecordFactory implements Serializable {
s s
.getLicense() .getLicense()
.stream() .stream()
.filter(Objects::nonNull)
.map(l -> XmlSerializationUtils.mapStructuredProperty("license", l)) .map(l -> XmlSerializationUtils.mapStructuredProperty("license", l))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -576,6 +596,7 @@ public class XmlRecordFactory implements Serializable {
ds ds
.getOdlanguages() .getOdlanguages()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("odlanguages", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("odlanguages", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -585,6 +606,7 @@ public class XmlRecordFactory implements Serializable {
ds ds
.getOdcontenttypes() .getOdcontenttypes()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("odcontenttypes", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("odcontenttypes", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -697,6 +719,7 @@ public class XmlRecordFactory implements Serializable {
ds ds
.getPolicies() .getPolicies()
.stream() .stream()
.filter(XmlRecordFactory::kvNotBlank)
.map(kv -> XmlSerializationUtils.mapKeyValue("policies", kv)) .map(kv -> XmlSerializationUtils.mapKeyValue("policies", kv))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -709,6 +732,7 @@ public class XmlRecordFactory implements Serializable {
ds ds
.getSubjects() .getSubjects()
.stream() .stream()
.filter(Objects::nonNull)
.map(sp -> XmlSerializationUtils.mapStructuredProperty("subjects", sp)) .map(sp -> XmlSerializationUtils.mapStructuredProperty("subjects", sp))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -735,6 +759,7 @@ public class XmlRecordFactory implements Serializable {
o o
.getAlternativeNames() .getAlternativeNames()
.stream() .stream()
.filter(Objects::nonNull)
.map(c -> XmlSerializationUtils.asXmlElement("alternativeNames", c.getValue())) .map(c -> XmlSerializationUtils.asXmlElement("alternativeNames", c.getValue()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -862,6 +887,7 @@ public class XmlRecordFactory implements Serializable {
p p
.getSubjects() .getSubjects()
.stream() .stream()
.filter(Objects::nonNull)
.map(sp -> XmlSerializationUtils.mapStructuredProperty("subject", sp)) .map(sp -> XmlSerializationUtils.mapStructuredProperty("subject", sp))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -912,7 +938,12 @@ public class XmlRecordFactory implements Serializable {
if (p.getFundingtree() != null) { if (p.getFundingtree() != null) {
metadata metadata
.addAll( .addAll(
p.getFundingtree().stream().map(ft -> ft.getValue()).collect(Collectors.toList())); p
.getFundingtree()
.stream()
.filter(Objects::nonNull)
.map(ft -> ft.getValue())
.collect(Collectors.toList()));
} }
break; break;
@ -923,6 +954,17 @@ public class XmlRecordFactory implements Serializable {
return metadata; return metadata;
} }
private String getAuthorPidType(String s) {
return XmlSerializationUtils
.escapeXml(s)
.replaceAll("\\W", "")
.replaceAll("\\d", "");
}
private static boolean kvNotBlank(KeyValue kv) {
return kv != null && StringUtils.isNotBlank(kv.getKey()) && StringUtils.isNotBlank(kv.getValue());
}
private void mapDatasourceType(List<String> metadata, final Qualifier dsType) { private void mapDatasourceType(List<String> metadata, final Qualifier dsType) {
metadata.add(XmlSerializationUtils.mapQualifier("datasourcetype", dsType)); metadata.add(XmlSerializationUtils.mapQualifier("datasourcetype", dsType));
@ -960,7 +1002,7 @@ public class XmlRecordFactory implements Serializable {
.add( .add(
XmlSerializationUtils.asXmlElement("coderepositoryurl", re.getCodeRepositoryUrl())); XmlSerializationUtils.asXmlElement("coderepositoryurl", re.getCodeRepositoryUrl()));
} }
if (re.getResulttype() != null & re.getResulttype().isBlank()) { if (re.getResulttype() != null && re.getResulttype().isBlank()) {
metadata.add(XmlSerializationUtils.mapQualifier("resulttype", re.getResulttype())); metadata.add(XmlSerializationUtils.mapQualifier("resulttype", re.getResulttype()));
} }
if (re.getCollectedfrom() != null) { if (re.getCollectedfrom() != null) {
@ -969,6 +1011,7 @@ public class XmlRecordFactory implements Serializable {
re re
.getCollectedfrom() .getCollectedfrom()
.stream() .stream()
.filter(XmlRecordFactory::kvNotBlank)
.map(kv -> XmlSerializationUtils.mapKeyValue("collectedfrom", kv)) .map(kv -> XmlSerializationUtils.mapKeyValue("collectedfrom", kv))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -986,10 +1029,10 @@ public class XmlRecordFactory implements Serializable {
if (isNotBlank(re.getOfficialname())) { if (isNotBlank(re.getOfficialname())) {
metadata.add(XmlSerializationUtils.asXmlElement("officialname", re.getOfficialname())); metadata.add(XmlSerializationUtils.asXmlElement("officialname", re.getOfficialname()));
} }
if (re.getDatasourcetype() != null & !re.getDatasourcetype().isBlank()) { if (re.getDatasourcetype() != null && !re.getDatasourcetype().isBlank()) {
mapDatasourceType(metadata, re.getDatasourcetype()); mapDatasourceType(metadata, re.getDatasourcetype());
} }
if (re.getOpenairecompatibility() != null & !re.getOpenairecompatibility().isBlank()) { if (re.getOpenairecompatibility() != null && !re.getOpenairecompatibility().isBlank()) {
metadata metadata
.add( .add(
XmlSerializationUtils XmlSerializationUtils
@ -1006,7 +1049,7 @@ public class XmlRecordFactory implements Serializable {
.add( .add(
XmlSerializationUtils.asXmlElement("legalshortname", re.getLegalshortname())); XmlSerializationUtils.asXmlElement("legalshortname", re.getLegalshortname()));
} }
if (re.getCountry() != null & !re.getCountry().isBlank()) { if (re.getCountry() != null && !re.getCountry().isBlank()) {
metadata.add(XmlSerializationUtils.mapQualifier("country", re.getCountry())); metadata.add(XmlSerializationUtils.mapQualifier("country", re.getCountry()));
} }
break; break;
@ -1020,10 +1063,10 @@ public class XmlRecordFactory implements Serializable {
if (isNotBlank(re.getAcronym())) { if (isNotBlank(re.getAcronym())) {
metadata.add(XmlSerializationUtils.asXmlElement("acronym", re.getAcronym())); metadata.add(XmlSerializationUtils.asXmlElement("acronym", re.getAcronym()));
} }
if (re.getContracttype() != null & !re.getContracttype().isBlank()) { if (re.getContracttype() != null && !re.getContracttype().isBlank()) {
metadata.add(XmlSerializationUtils.mapQualifier("contracttype", re.getContracttype())); metadata.add(XmlSerializationUtils.mapQualifier("contracttype", re.getContracttype()));
} }
if (re.getFundingtree() != null & contexts != null) { if (re.getFundingtree() != null && contexts != null) {
metadata metadata
.addAll( .addAll(
re re
@ -1091,12 +1134,12 @@ public class XmlRecordFactory implements Serializable {
.add( .add(
XmlSerializationUtils.mapQualifier("accessright", instance.getAccessright())); XmlSerializationUtils.mapQualifier("accessright", instance.getAccessright()));
} }
if (instance.getCollectedfrom() != null) { if (instance.getCollectedfrom() != null && kvNotBlank(instance.getCollectedfrom())) {
fields fields
.add( .add(
XmlSerializationUtils.mapKeyValue("collectedfrom", instance.getCollectedfrom())); XmlSerializationUtils.mapKeyValue("collectedfrom", instance.getCollectedfrom()));
} }
if (instance.getHostedby() != null) { if (instance.getHostedby() != null && kvNotBlank(instance.getHostedby())) {
fields.add(XmlSerializationUtils.mapKeyValue("hostedby", instance.getHostedby())); fields.add(XmlSerializationUtils.mapKeyValue("hostedby", instance.getHostedby()));
} }
if (instance.getDateofacceptance() != null if (instance.getDateofacceptance() != null