lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

Merge pull request 'dhp-schema upgrade & provision mapping' (#498) from beta_provision_alignment_9.0.0 into beta 

Reviewed-on: D-Net/dnet-hadoop#498
This commit is contained in:
Claudio Atzori 2024-10-28 10:03:24 +01:00
parent 67e37f41fb 32fa579b80
commit dffa376eb6
4 changed files with 23 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ModelHardLimits.java
View File

@ -12,6 +12,7 @@ public class ModelHardLimits {
public static final int MAX_EXTERNAL_ENTITIES = 50;
public static final int MAX_AUTHORS = 200;
public static final int MAX_RELATED_AUTHORS = 20;
public static final int MAX_AUTHOR_FULLNAME_LENGTH = 1000;
public static final int MAX_TITLE_LENGTH = 5000;
public static final int MAX_TITLES = 10;

4
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase1.java
View File

@ -3,6 +3,7 @@ package eu.dnetlib.dhp.oa.provision;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@ -167,8 +168,9 @@ public class CreateRelatedEntitiesJob_phase1 {
result
.getDescription()
.stream()
.findFirst()
.filter(d -> Objects.nonNull(d.getValue()))
.map(Field::getValue)
.max(Comparator.comparingInt(String::length))
.ifPresent(
d -> re.setDescription(StringUtils.left(d, ModelHardLimits.MAX_RELATED_ABSTRACT_LENGTH)));
}

19
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java
View File

@ -5,6 +5,7 @@ import java.io.StringReader;
import java.util.*;
import java.util.stream.Collectors;
import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits;
import org.apache.commons.lang3.StringUtils;
import org.dom4j.Document;
import org.dom4j.DocumentException;
@ -150,6 +151,12 @@ public class ProvisionModelSupport {
rr.setPublisher(re.getPublisher());
rr.setResulttype(mapQualifier(re.getResulttype()));
rr.setTitle(Optional.ofNullable(re.getTitle()).map(StructuredProperty::getValue).orElse(null));
rr.setDescription(StringUtils.left(re.getDescription(), ModelHardLimits.MAX_RELATED_ABSTRACT_LENGTH));
rr.setAuthor(Optional.ofNullable(re.getAuthor())
.map(aa -> aa.stream()
.limit(ModelHardLimits.MAX_RELATED_AUTHORS)
.collect(Collectors.toList()))
.orElse(null));
if (relation.getValidated() == null) {
relation.setValidated(false);
@ -378,6 +385,7 @@ public class ProvisionModelSupport {
rs.setPubliclyFunded(r.getPubliclyFunded());
rs.setTransformativeAgreement(r.getTransformativeAgreement());
rs.setExternalReference(mapExternalReference(r.getExternalReference()));
rs.setBestinstancetype(mapQualifier(r.getBestInstancetype()));
rs.setInstance(mapInstances(r.getInstance()));
if (r instanceof Publication) {
@ -667,14 +675,23 @@ public class ProvisionModelSupport {
}
private static List<Author> asAuthor(List<eu.dnetlib.dhp.schema.oaf.Author> authorList) {
return asAuthor(authorList, ModelHardLimits.MAX_AUTHORS);
}
private static List<Author> asAuthor(List<eu.dnetlib.dhp.schema.oaf.Author> authorList, int maxAuthors) {
return Optional
.ofNullable(authorList)
.map(
authors -> authors
.stream()
.limit(maxAuthors)
.map(
a -> Author
.newInstance(a.getFullname(), a.getName(), a.getSurname(), a.getRank(), asPid(a.getPid())))
.newInstance(
StringUtils.left(a.getFullname(), ModelHardLimits.MAX_AUTHOR_FULLNAME_LENGTH),
a.getName(),
a.getSurname(),
a.getRank(), asPid(a.getPid())))
.collect(Collectors.toList()))
.orElse(null);
}

2
pom.xml
View File

@ -937,7 +937,7 @@
<commons.logging.version>1.1.3</commons.logging.version>
<commons-validator.version>1.7</commons-validator.version>
<dateparser.version>1.0.7</dateparser.version>
<dhp-schemas.version>[8.0.1]</dhp-schemas.version>
<dhp-schemas.version>[9.0.0]</dhp-schemas.version>
<dhp.cdh.version>cdh5.9.2</dhp.cdh.version>
<dhp.commons.lang.version>3.5</dhp.commons.lang.version>
<dhp.guava.version>11.0.2</dhp.guava.version>