D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

[graph provision] person serialisation, limit the number of authorships and coauthorships before expanding the payloads

This commit is contained in:
Claudio Atzori 2024-09-24 10:57:20 +02:00
parent 0e89d4a1cf
commit d1cadc77c9
2 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ModelHardLimits.java
View File

@ -25,11 +25,11 @@ public class ModelHardLimits {
public static final int MAX_ABSTRACT_LENGTH = 150000;
public static final int MAX_RELATED_ABSTRACT_LENGTH = 500;
public static final int MAX_INSTANCES = 10;
public static final Map<String, Integer> MAX_RELATIONS_BY_RELCLASS = Maps.newHashMap();
public static final Map<String, Long> MAX_RELATIONS_BY_RELCLASS = Maps.newHashMap();
static {
MAX_RELATIONS_BY_RELCLASS.put(ModelConstants.PERSON_PERSON_HASCOAUTHORED, 500);
MAX_RELATIONS_BY_RELCLASS.put(ModelConstants.RESULT_PERSON_HASAUTHORED, 500);
MAX_RELATIONS_BY_RELCLASS.put(ModelConstants.PERSON_PERSON_HASCOAUTHORED, 500L);
MAX_RELATIONS_BY_RELCLASS.put(ModelConstants.RESULT_PERSON_HASAUTHORED, 500L);
}
public static String getCollectionName(String format) {

12
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java
View File

@ -149,8 +149,8 @@ public class PayloadConverterJob {
}
/**
This function iterates through the RelatedEntityWrapper(s) associated to the JoinedEntity and rules out
those exceeding the maximum allowed frequency defined in eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits#MAX_RELATIONS_BY_RELCLASS
* This function iterates through the RelatedEntityWrapper(s) associated to the JoinedEntity and rules out
* those exceeding the maximum allowed frequency defined in eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits#MAX_RELATIONS_BY_RELCLASS
*/
private static JoinedEntity pruneRelatedEntities(JoinedEntity je) {
Map<String, Long> freqs = Maps.newHashMap();
@ -159,9 +159,11 @@ public class PayloadConverterJob {
if (je.getLinks() != null) {
je.getLinks().forEach(link -> {
final String relClass = link.getRelation().getRelClass();
Long count = freqs.putIfAbsent(relClass, 0L);
if (Objects.isNull(count) || (MAX_RELATIONS_BY_RELCLASS.containsKey(relClass)
&& count <= MAX_RELATIONS_BY_RELCLASS.get(relClass))) {
final Long count = freqs.getOrDefault(relClass, Long.MAX_VALUE);
final Long max = MAX_RELATIONS_BY_RELCLASS.getOrDefault(relClass, Long.MAX_VALUE);
if (count <= max) {
rew.add(link);
freqs.put(relClass, freqs.get(relClass) + 1);
}