D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

[graph provision] person serialisation, limit the number of authorships and coauthorships before expanding the payloads

This commit is contained in:
Claudio Atzori 2024-09-24 10:57:20 +02:00
parent 0e89d4a1cf
commit d1cadc77c9
2 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ModelHardLimits.java
View File

@ -25,11 +25,11 @@ public class ModelHardLimits {
public static final int MAX_ABSTRACT_LENGTH = 150000; public static final int MAX_ABSTRACT_LENGTH = 150000;
public static final int MAX_RELATED_ABSTRACT_LENGTH = 500; public static final int MAX_RELATED_ABSTRACT_LENGTH = 500;
public static final int MAX_INSTANCES = 10; public static final int MAX_INSTANCES = 10;
public static final Map<String, Integer> MAX_RELATIONS_BY_RELCLASS = Maps.newHashMap(); public static final Map<String, Long> MAX_RELATIONS_BY_RELCLASS = Maps.newHashMap();
static { static {
MAX_RELATIONS_BY_RELCLASS.put(ModelConstants.PERSON_PERSON_HASCOAUTHORED, 500); MAX_RELATIONS_BY_RELCLASS.put(ModelConstants.PERSON_PERSON_HASCOAUTHORED, 500L);
MAX_RELATIONS_BY_RELCLASS.put(ModelConstants.RESULT_PERSON_HASAUTHORED, 500); MAX_RELATIONS_BY_RELCLASS.put(ModelConstants.RESULT_PERSON_HASAUTHORED, 500L);
} }
public static String getCollectionName(String format) { public static String getCollectionName(String format) {

12
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java
View File

@ -149,8 +149,8 @@ public class PayloadConverterJob {
} }
/** /**
This function iterates through the RelatedEntityWrapper(s) associated to the JoinedEntity and rules out * This function iterates through the RelatedEntityWrapper(s) associated to the JoinedEntity and rules out
those exceeding the maximum allowed frequency defined in eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits#MAX_RELATIONS_BY_RELCLASS * those exceeding the maximum allowed frequency defined in eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits#MAX_RELATIONS_BY_RELCLASS
*/ */
private static JoinedEntity pruneRelatedEntities(JoinedEntity je) { private static JoinedEntity pruneRelatedEntities(JoinedEntity je) {
Map<String, Long> freqs = Maps.newHashMap(); Map<String, Long> freqs = Maps.newHashMap();
@ -159,9 +159,11 @@ public class PayloadConverterJob {
if (je.getLinks() != null) { if (je.getLinks() != null) {
je.getLinks().forEach(link -> { je.getLinks().forEach(link -> {
final String relClass = link.getRelation().getRelClass(); final String relClass = link.getRelation().getRelClass();
Long count = freqs.putIfAbsent(relClass, 0L);
if (Objects.isNull(count) || (MAX_RELATIONS_BY_RELCLASS.containsKey(relClass) final Long count = freqs.getOrDefault(relClass, Long.MAX_VALUE);
&& count <= MAX_RELATIONS_BY_RELCLASS.get(relClass))) { final Long max = MAX_RELATIONS_BY_RELCLASS.getOrDefault(relClass, Long.MAX_VALUE);
if (count <= max) {
rew.add(link); rew.add(link);
freqs.put(relClass, freqs.get(relClass) + 1); freqs.put(relClass, freqs.get(relClass) + 1);
} }