diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml
index 44165995d..688b54f1d 100644
--- a/dhp-build/dhp-build-assembly-resources/pom.xml
+++ b/dhp-build/dhp-build-assembly-resources/pom.xml
@@ -6,7 +6,7 @@
eu.dnetlib.dhp
dhp-build
- 1.2.5-SNAPSHOT
+ 2.0.0-SNAPSHOT
dhp-build-assembly-resources
diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml
index 7579bdf45..ebb917437 100644
--- a/dhp-build/dhp-build-properties-maven-plugin/pom.xml
+++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml
@@ -6,7 +6,7 @@
eu.dnetlib.dhp
dhp-build
- 1.2.5-SNAPSHOT
+ 2.0.0-SNAPSHOT
dhp-build-properties-maven-plugin
diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml
index 9040ea94e..21a277f8f 100644
--- a/dhp-build/pom.xml
+++ b/dhp-build/pom.xml
@@ -4,7 +4,7 @@
eu.dnetlib.dhp
dhp
- 1.2.5-SNAPSHOT
+ 2.0.0-SNAPSHOT
dhp-build
pom
diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml
index 6df11f4ea..377db5681 100644
--- a/dhp-common/pom.xml
+++ b/dhp-common/pom.xml
@@ -5,7 +5,7 @@
eu.dnetlib.dhp
dhp
- 1.2.5-SNAPSHOT
+ 2.0.0-SNAPSHOT
../pom.xml
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/action/ReadDatasourceMasterDuplicateFromDB.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/action/ReadDatasourceMasterDuplicateFromDB.java
index 5d39216f1..2d292a139 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/action/ReadDatasourceMasterDuplicateFromDB.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/action/ReadDatasourceMasterDuplicateFromDB.java
@@ -1,6 +1,8 @@
package eu.dnetlib.dhp.common.action;
+import static eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory.*;
+
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
@@ -19,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.common.DbClient;
import eu.dnetlib.dhp.common.action.model.MasterDuplicate;
-import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
public class ReadDatasourceMasterDuplicateFromDB {
@@ -59,8 +60,8 @@ public class ReadDatasourceMasterDuplicateFromDB {
final String masterId = rs.getString("masterId");
final String masterName = rs.getString("masterName");
- md.setDuplicateId(OafMapperUtils.createOpenaireId(10, duplicateId, true));
- md.setMasterId(OafMapperUtils.createOpenaireId(10, masterId, true));
+ md.setDuplicateId(createOpenaireId(10, duplicateId, true));
+ md.setMasterId(createOpenaireId(10, masterId, true));
md.setMasterName(masterName);
return md;
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java
index aa3c857cf..ff6fcde94 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java
@@ -121,10 +121,12 @@ public class AuthorMerger {
}
public static String pidToComparableString(StructuredProperty pid) {
- final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase()
- : "";
- return (pid.getQualifier() != null ? classid : "")
- + (pid.getValue() != null ? pid.getValue().toLowerCase() : "");
+ return pid.toComparableString();
+ /*
+ * final String classid = pid.getQualifier().getClassid() != null ?
+ * pid.getQualifier().getClassid().toLowerCase() : ""; return (pid.getQualifier() != null ? classid : "") +
+ * (pid.getValue() != null ? pid.getValue().toLowerCase() : "");
+ */
}
public static int countAuthorsPids(List authors) {
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java
index bae28447e..1280d6fde 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java
@@ -10,8 +10,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
-import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
-import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
@@ -33,6 +31,8 @@ import com.jayway.jsonpath.Option;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.HdfsSupport;
import eu.dnetlib.dhp.schema.oaf.*;
+import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
+import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils;
import scala.Tuple2;
/**
@@ -120,7 +120,7 @@ public class GroupEntitiesSparkJob {
private Entity mergeAndGet(Entity b, Entity a) {
if (Objects.nonNull(a) && Objects.nonNull(b)) {
- return MergeUtils.mergeEntities(b, a);
+ return MergeUtils.merge(b, a);
}
return Objects.isNull(a) ? b : a;
}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java
new file mode 100644
index 000000000..d2ef9fa7b
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java
@@ -0,0 +1,252 @@
+
+package eu.dnetlib.dhp.schema.common;
+
+import eu.dnetlib.dhp.schema.oaf.AccessRight;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Qualifier;
+
+public class ModelConstants {
+
+ private ModelConstants() {}
+
+ public static final String ORCID = "orcid";
+ public static final String ORCID_PENDING = "orcid_pending";
+ public static final String ORCID_CLASSNAME = "Open Researcher and Contributor ID";
+ public static final String ORCID_DS = ORCID.toUpperCase();
+
+ public static final String CROSSREF_ID = "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2";
+
+ public static final String CROSSREF_NAME = "Crossref";
+ public static final String DATACITE_ID = "10|openaire____::9e3be59865b2c1c335d32dae2fe7b254";
+
+ public static final String ZENODO_OD_ID = "10|opendoar____::358aee4cc897452c00244351e4d91f69";
+ public static final String ZENODO_R3_ID = "10|re3data_____::7b0ad08687b2c960d5aeef06f811d5e6";
+
+ public static final String EUROPE_PUBMED_CENTRAL_ID = "10|opendoar____::8b6dd7db9af49e67306feb59a8bdc52c";
+ public static final String PUBMED_CENTRAL_ID = "10|opendoar____::eda80a3d5b344bc40f3bc04f65b7a357";
+ public static final String ARXIV_ID = "10|opendoar____::6f4922f45568161a8cdf4ad2299f6d23";
+ public static final String ROHUB_ID = "10|fairsharing_::1b69ebedb522700034547abc5652ffac";
+
+ public static final String OPENORGS_NAME = "OpenOrgs Database";
+
+ public static final String OPENOCITATIONS_NAME = "OpenCitations";
+ public static final String OPENOCITATIONS_ID = "10|openaire____::c06df618c5de1c786535ccf3f8b7b059";
+
+ public static final String OPEN_APC_NAME = "OpenAPC Global Initiative";
+ public static final String OPEN_APC_ID = "10|apc_________::e2b1600b229fc30663c8a1f662debddf";
+
+ // VOCABULARY VALUE
+ public static final String ACCESS_RIGHT_OPEN = "OPEN";
+ public static final String ACCESS_RIGHT_EMBARGO = "EMBARGO";
+ public static final String ACCESS_RIGHT_CLOSED = "CLOSED";
+
+ public static final String DNET_SUBJECT_KEYWORD = "keyword";
+
+ public static final String DNET_SUBJECT_FOS_CLASSID = "FOS";
+
+ public static final String DNET_SUBJECT_FOS_CLASSNAME = "Fields of Science and Technology classification";
+
+ public static final String DNET_SUBJECT_TYPOLOGIES = "dnet:subject_classification_typologies";
+ public static final String DNET_RESULT_TYPOLOGIES = "dnet:result_typologies";
+ public static final String DNET_PUBLICATION_RESOURCE = "dnet:publication_resource";
+ public static final String DNET_ACCESS_MODES = "dnet:access_modes";
+ public static final String DNET_LANGUAGES = "dnet:languages";
+ public static final String DNET_PID_TYPES = "dnet:pid_types";
+ public static final String DNET_DATACITE_DATE = "dnet:dataCite_date";
+ public static final String DNET_DATACITE_TITLE = "dnet:dataCite_title";
+ public static final String DNET_DATA_CITE_RESOURCE = "dnet:dataCite_resource";
+ public static final String DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions";
+ public static final String DNET_COUNTRY_TYPE = "dnet:countries";
+ public static final String DNET_REVIEW_LEVELS = "dnet:review_levels";
+ public static final String DNET_PROGRAMMING_LANGUAGES = "dnet:programming_languages";
+ public static final String DNET_EXTERNAL_REFERENCE_TYPE = "dnet:externalReference_typologies";
+ public static final String DNET_RELATION_RELTYPE = "dnet:relation_relType";
+ public static final String DNET_RELATION_SUBRELTYPE = "dnet:relation_subRelType";
+ public static final String DNET_RELATION_RELCLASS = "dnet:relation_relClass";
+
+ public static final String PEER_REVIEWED_CLASSNAME = "nonPeerReviewed";
+ public static final String NON_PEER_REVIEWED_CLASSNAME = "nonPeerReviewed";
+ public static final String PEER_REVIEWED_CLASSID = "0001";
+ public static final String NON_PEER_REVIEWED_CLASSID = "0002";
+
+ public static final String SYSIMPORT_CROSSWALK_REPOSITORY = "sysimport:crosswalk:repository";
+ public static final String SYSIMPORT_CROSSWALK_ENTITYREGISTRY = "sysimport:crosswalk:entityregistry";
+ public static final String SYSIMPORT_ACTIONSET = "sysimport:actionset";
+ public static final String SYSIMPORT_ORCID_NO_DOI = "sysimport:actionset:orcidworks-no-doi";
+
+ public static final String USER_CLAIM = "user:claim";
+ public static final String HARVESTED = "Harvested";
+
+ public static final String PROVENANCE_DEDUP = "sysimport:dedup";
+ public static final String PROVENANCE_ENRICH = "sysimport:enrich";
+
+
+ public static final Qualifier PROVENANCE_ACTION_SET_QUALIFIER = qualifier(
+ SYSIMPORT_ACTIONSET, SYSIMPORT_ACTIONSET, DNET_PROVENANCE_ACTIONS);
+
+ public static final String DATASET_RESULTTYPE_CLASSID = "dataset";
+ public static final String PUBLICATION_RESULTTYPE_CLASSID = "publication";
+ public static final String SOFTWARE_RESULTTYPE_CLASSID = "software";
+ public static final String ORP_RESULTTYPE_CLASSID = "other";
+
+ public static final String RESULT_RESULT = "resultResult"; // relType
+ /**
+ * @deprecated Use {@link ModelConstants#RELATIONSHIP} instead.
+ */
+ @Deprecated
+ public static final String PUBLICATION_DATASET = "publicationDataset"; // subreltype
+
+ public static final String SUPPLEMENT = "supplement"; // subreltype
+ public static final String IS_SUPPLEMENT_TO = "IsSupplementTo";
+ public static final String IS_SUPPLEMENTED_BY = "IsSupplementedBy";
+
+ public static final String PART = "part"; // subreltype
+ public static final String IS_PART_OF = "IsPartOf";
+ public static final String HAS_PART = "HasPart";
+
+ public static final String RELATIONSHIP = "relationship"; // subreltype
+
+ public static final String IS_RELATED_TO = "IsRelatedTo";
+ public static final String IS_IDENTICAL_TO = "IsIdenticalTo";
+
+ public static final String REFERENCES = "References";
+ public static final String IS_REFERENCED_BY = "IsReferencedBy";
+ public static final String CONTINUES = "Continues";
+ public static final String IS_CONTINUED_BY = "IsContinuedBy";
+ public static final String DOCUMENTS = "Documents";
+ public static final String IS_DOCUMENTED_BY = "IsDocumentedBy";
+ public static final String IS_SOURCE_OF = "IsSourceOf";
+ public static final String IS_DERIVED_FROM = "IsDerivedFrom";
+ public static final String COMPILES = "Compiles";
+ public static final String IS_COMPILED_BY = "IsCompiledBy";
+ public static final String DESCRIBES = "Describes";
+ public static final String IS_DESCRIBED_BY = "IsDescribedBy";
+ public static final String IS_METADATA_FOR = "IsMetadataFor";
+ public static final String IS_METADATA_OF = "IsMetadataOf";
+ public static final String HAS_ASSOCIATION_WITH = "HasAssociationWith";
+ public static final String IS_REQUIRED_BY = "IsRequiredBy";
+ public static final String REQUIRES = "Requires";
+
+
+
+ public static final String CITATION = "citation"; // subreltype
+ public static final String CITES = "Cites";
+ public static final String IS_CITED_BY = "IsCitedBy";
+
+ public static final String REVIEW = "review"; // subreltype
+ public static final String REVIEWS = "Reviews";
+ public static final String IS_REVIEWED_BY = "IsReviewedBy";
+
+ public static final String VERSION = "version"; // subreltype
+ public static final String IS_VERSION_OF = "IsVersionOf";
+ public static final String HAS_VERSION = "HasVersion";
+ public static final String IS_PREVIOUS_VERSION_OF = "IsPreviousVersionOf";
+ public static final String IS_NEW_VERSION_OF = "IsNewVersionOf";
+ public static final String IS_VARIANT_FORM_OF = "IsVariantFormOf";
+ public static final String IS_ORIGINAL_FORM_OF = "IsOriginalFormOf";
+ public static final String IS_OBSOLETED_BY = "IsObsoletedBy";
+ public static final String OBSOLETES = "Obsoletes";
+
+ public static final String RESULT_PROJECT = "resultProject"; // relType
+ public static final String OUTCOME = "outcome"; // subreltype
+ public static final String IS_PRODUCED_BY = "isProducedBy";
+ public static final String PRODUCES = "produces";
+
+ public static final String DATASOURCE_ORGANIZATION = "datasourceOrganization"; // relType
+ public static final String PROVISION = "provision"; // subreltype
+ public static final String IS_PROVIDED_BY = "isProvidedBy";
+ public static final String PROVIDES = "provides";
+
+ public static final String PROJECT_ORGANIZATION = "projectOrganization"; // relType
+ public static final String PARTICIPATION = "participation"; // subreltype
+ public static final String HAS_PARTICIPANT = "hasParticipant";
+ public static final String IS_PARTICIPANT = "isParticipant";
+
+ public static final String RESULT_ORGANIZATION = "resultOrganization"; // relType
+ public static final String AFFILIATION = "affiliation"; // subreltype
+ public static final String IS_AUTHOR_INSTITUTION_OF = "isAuthorInstitutionOf";
+ public static final String HAS_AUTHOR_INSTITUTION = "hasAuthorInstitution";
+
+ public static final String ORG_ORG_RELTYPE = "organizationOrganization"; // relType
+ public static final String IS_PARENT_OF = "IsParentOf";
+ public static final String IS_CHILD_OF = "IsChildOf";
+
+ public static final String DEDUP = "dedup"; // subreltype
+ public static final String MERGES = "merges";
+ public static final String IS_MERGED_IN = "isMergedIn";
+
+ public static final String SIMILARITY = "similarity"; // subreltype
+ public static final String IS_SIMILAR_TO = "isSimilarTo";
+ public static final String IS_AMONG_TOP_N_SIMILAR_DOCS = "IsAmongTopNSimilarDocuments";
+ public static final String HAS_AMONG_TOP_N_SIMILAR_DOCS = "HasAmongTopNSimilarDocuments";
+
+ public static final String IS_DIFFERENT_FROM = "isDifferentFrom";
+
+ public static final String UNKNOWN = "UNKNOWN";
+ public static final String NOT_AVAILABLE = "not available";
+
+ public static final Qualifier PUBLICATION_DEFAULT_RESULTTYPE = qualifier(
+ PUBLICATION_RESULTTYPE_CLASSID, PUBLICATION_RESULTTYPE_CLASSID,
+ DNET_RESULT_TYPOLOGIES);
+
+ public static final Qualifier DATASET_DEFAULT_RESULTTYPE = qualifier(
+ DATASET_RESULTTYPE_CLASSID, DATASET_RESULTTYPE_CLASSID,
+ DNET_RESULT_TYPOLOGIES);
+
+ public static final Qualifier SOFTWARE_DEFAULT_RESULTTYPE = qualifier(
+ SOFTWARE_RESULTTYPE_CLASSID, SOFTWARE_RESULTTYPE_CLASSID,
+ DNET_RESULT_TYPOLOGIES);
+
+ public static final Qualifier ORP_DEFAULT_RESULTTYPE = qualifier(
+ ORP_RESULTTYPE_CLASSID, ORP_RESULTTYPE_CLASSID,
+ DNET_RESULT_TYPOLOGIES);
+
+ public static final Qualifier REPOSITORY_PROVENANCE_ACTIONS = qualifier(
+ SYSIMPORT_CROSSWALK_REPOSITORY, SYSIMPORT_CROSSWALK_REPOSITORY,
+ DNET_PROVENANCE_ACTIONS);
+
+ public static final Qualifier ENTITYREGISTRY_PROVENANCE_ACTION = qualifier(
+ SYSIMPORT_CROSSWALK_ENTITYREGISTRY, SYSIMPORT_CROSSWALK_ENTITYREGISTRY,
+ DNET_PROVENANCE_ACTIONS);
+
+ public static final String UNKNOWN_REPOSITORY_ORIGINALID = "openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18";
+ public static final KeyValue UNKNOWN_REPOSITORY = keyValue(
+ "10|openaire____::55045bd2a65019fd8e6741a755395c8c", "Unknown Repository");
+
+ public static final Qualifier UNKNOWN_COUNTRY = qualifier(UNKNOWN, "Unknown", DNET_COUNTRY_TYPE);
+
+ public static final Qualifier MAIN_TITLE_QUALIFIER = qualifier(
+ "main title", "main title", DNET_DATACITE_TITLE);
+
+ public static final Qualifier ALTERNATIVE_TITLE_QUALIFIER = qualifier(
+ "alternative title", "alternative title", DNET_DATACITE_TITLE);
+
+ private static final Qualifier SUBTITLE_QUALIFIER = qualifier("subtitle", "subtitle", DNET_DATACITE_TITLE);
+
+ public static final AccessRight OPEN_ACCESS_RIGHT() {
+
+ final AccessRight result = new AccessRight();
+ result.setClassid(ACCESS_RIGHT_OPEN);
+ result.setClassid(ACCESS_RIGHT_OPEN);
+ result.setSchemeid(ModelConstants.DNET_ACCESS_MODES);
+ return result;
+ }
+
+ private static Qualifier qualifier(
+ final String classid,
+ final String classname,
+ final String schemeid) {
+ final Qualifier q = new Qualifier();
+ q.setClassid(classid);
+ q.setClassname(classname);
+ q.setSchemeid(schemeid);
+ return q;
+ }
+
+ private static KeyValue keyValue(final String key, final String value) {
+ final KeyValue kv = new KeyValue();
+ kv.setKey(key);
+ kv.setValue(value);
+ return kv;
+ }
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/AccessRightComparator.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/AccessRightComparator.java
index 6efd1c3dd..3e80bd95b 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/AccessRightComparator.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/AccessRightComparator.java
@@ -1,10 +1,10 @@
package eu.dnetlib.dhp.schema.oaf.common;
-import eu.dnetlib.dhp.schema.oaf.Qualifier;
-
import java.util.Comparator;
+import eu.dnetlib.dhp.schema.oaf.Qualifier;
+
public class AccessRightComparator implements Comparator {
@Override
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/ModelSupport.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/ModelSupport.java
index 5d03dfb4a..3ea391bd4 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/ModelSupport.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/ModelSupport.java
@@ -1,12 +1,8 @@
package eu.dnetlib.dhp.schema.oaf.common;
-import com.github.sisyphsu.dateparser.DateParserUtils;
-import com.google.common.collect.Maps;
-
-import eu.dnetlib.dhp.schema.oaf.*;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.lang3.StringUtils;
+import static com.google.common.base.Preconditions.checkArgument;
+import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -18,8 +14,13 @@ import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
-import static com.google.common.base.Preconditions.checkArgument;
-import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.lang3.StringUtils;
+
+import com.github.sisyphsu.dateparser.DateParserUtils;
+import com.google.common.collect.Maps;
+
+import eu.dnetlib.dhp.schema.oaf.*;
/** Oaf model utility methods. */
public class ModelSupport {
@@ -129,7 +130,6 @@ public class ModelSupport {
set(relationInverseMap, RESULT_RESULT, RELATIONSHIP, HAS_ASSOCIATION_WITH, HAS_ASSOCIATION_WITH);
set(relationInverseMap, RESULT_RESULT, RELATIONSHIP, IS_REQUIRED_BY, REQUIRES);
-
set(relationInverseMap, RESULT_RESULT, VERSION, IS_PREVIOUS_VERSION_OF, IS_NEW_VERSION_OF);
set(relationInverseMap, RESULT_RESULT, VERSION, IS_VARIANT_FORM_OF, IS_ORIGINAL_FORM_OF);
set(relationInverseMap, RESULT_RESULT, VERSION, IS_OBSOLETED_BY, OBSOLETES);
@@ -138,22 +138,23 @@ public class ModelSupport {
set(relationInverseMap, RESULT_RESULT, REVIEW, IS_REVIEWED_BY, REVIEWS);
}
- private static void set(Map relationInverseMap, String relType, String subRelType, String relClass, String inverseRelClass) {
+ private static void set(Map relationInverseMap, String relType, String subRelType,
+ String relClass, String inverseRelClass) {
relationInverseMap
- .put(
- rel(relType, subRelType, relClass), new RelationInverse()
- .setInverseRelClass(inverseRelClass)
- .setRelClass(relClass)
- .setRelType(relType)
- .setSubReltype(subRelType));
+ .put(
+ rel(relType, subRelType, relClass), new RelationInverse()
+ .setInverseRelClass(inverseRelClass)
+ .setRelClass(relClass)
+ .setRelType(relType)
+ .setSubReltype(subRelType));
if (!relClass.equals(inverseRelClass)) {
relationInverseMap
- .put(
- rel(relType, subRelType, inverseRelClass), new RelationInverse()
- .setInverseRelClass(relClass)
- .setRelClass(inverseRelClass)
- .setRelType(relType)
- .setSubReltype(subRelType));
+ .put(
+ rel(relType, subRelType, inverseRelClass), new RelationInverse()
+ .setInverseRelClass(relClass)
+ .setRelClass(inverseRelClass)
+ .setRelType(relType)
+ .setSubReltype(subRelType));
}
}
@@ -164,25 +165,26 @@ public class ModelSupport {
*/
public static RelationInverse findInverse(String encoding) {
return ModelSupport.relationInverseMap
- .entrySet()
- .stream()
- .filter(r -> encoding.equalsIgnoreCase(r.getKey()))
- .findFirst()
- .map(r -> r.getValue())
- .orElseThrow(() -> new IllegalArgumentException("invalid relationship: " + encoding));
+ .entrySet()
+ .stream()
+ .filter(r -> encoding.equalsIgnoreCase(r.getKey()))
+ .findFirst()
+ .map(r -> r.getValue())
+ .orElseThrow(() -> new IllegalArgumentException("invalid relationship: " + encoding));
}
/**
* Helper method: fina a relation filtering by a relation name
- * @param relationName
+ * @param relationName
* @return
*/
public static RelationInverse findRelation(final String relationName) {
- return relationInverseMap.values()
- .stream()
- .filter(r -> relationName.equalsIgnoreCase(r.getRelClass()))
- .findFirst()
- .orElse(null);
+ return relationInverseMap
+ .values()
+ .stream()
+ .filter(r -> relationName.equalsIgnoreCase(r.getRelClass()))
+ .findFirst()
+ .orElse(null);
}
/**
@@ -207,6 +209,10 @@ public class ModelSupport {
return idPrefixMap.get(clazz);
}
+ public static Boolean sameClass(X left, Y right, Class superClazz) {
+ return isSubClass(left, superClazz) && isSubClass(right, superClazz);
+ }
+
/**
* Checks subclass-superclass relationship.
*
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/RefereedComparator.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/RefereedComparator.java
index a1d712385..75e29e176 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/RefereedComparator.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/common/RefereedComparator.java
@@ -1,11 +1,11 @@
package eu.dnetlib.dhp.schema.oaf.common;
+import java.util.Comparator;
+
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.Qualifier;
-import java.util.Comparator;
-
public class RefereedComparator implements Comparator {
@Override
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java
index dc3b8e888..fff9ac885 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java
@@ -1,16 +1,7 @@
package eu.dnetlib.dhp.schema.oaf.utils;
-import com.github.sisyphsu.dateparser.DateParserUtils;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-
-import eu.dnetlib.dhp.schema.oaf.*;
-import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
-import me.xuender.unidecode.Unidecode;
-import org.apache.commons.lang3.StringUtils;
+import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.getProvenance;
import java.time.LocalDate;
import java.time.ZoneId;
@@ -21,7 +12,17 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.getProvenance;
+import org.apache.commons.lang3.StringUtils;
+
+import com.github.sisyphsu.dateparser.DateParserUtils;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.*;
+import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
+import me.xuender.unidecode.Unidecode;
public class GraphCleaningFunctions extends CleaningFunctions {
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/IdentifierFactory.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/IdentifierFactory.java
index cba65b02a..0db1e1b63 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/IdentifierFactory.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/IdentifierFactory.java
@@ -12,7 +12,6 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang3.StringUtils;
@@ -20,6 +19,7 @@ import com.google.common.collect.HashBiMap;
import com.google.common.collect.Maps;
import eu.dnetlib.dhp.schema.oaf.*;
+import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
/**
* Factory class for OpenAIRE identifiers in the Graph
@@ -268,7 +268,7 @@ public class IdentifierFactory implements Serializable {
.append(ID_PREFIX_SEPARATOR)
.append(createPrefix(pidType))
.append(ID_SEPARATOR)
- .append(md5 ? md5(pidValue) : pidValue)
+ .append(md5 ? ModelSupport.md5(pidValue) : pidValue)
.toString();
}
@@ -281,13 +281,36 @@ public class IdentifierFactory implements Serializable {
return prefix.substring(0, ID_PREFIX_LEN);
}
- public static String md5(final String s) {
- try {
- final MessageDigest md = MessageDigest.getInstance("MD5");
- md.update(s.getBytes(StandardCharsets.UTF_8));
- return new String(Hex.encodeHex(md.digest()));
- } catch (final Exception e) {
+ public static String createOpenaireId(
+ final int prefix,
+ final String originalId,
+ final boolean to_md5) {
+ if (StringUtils.isBlank(originalId)) {
return null;
+ } else if (to_md5) {
+ final String nsPrefix = StringUtils.substringBefore(originalId, "::");
+ final String rest = StringUtils.substringAfter(originalId, "::");
+ return String.format("%s|%s::%s", prefix, nsPrefix, ModelSupport.md5(rest));
+ } else {
+ return String.format("%s|%s", prefix, originalId);
+ }
+ }
+
+ public static String createOpenaireId(
+ final String type,
+ final String originalId,
+ final boolean to_md5) {
+ switch (type) {
+ case "datasource":
+ return createOpenaireId(10, originalId, to_md5);
+ case "organization":
+ return createOpenaireId(20, originalId, to_md5);
+ case "person":
+ return createOpenaireId(30, originalId, to_md5);
+ case "project":
+ return createOpenaireId(40, originalId, to_md5);
+ default:
+ return createOpenaireId(50, originalId, to_md5);
}
}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java
index ae01a6a79..7f148a4c8 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java
@@ -1,716 +1,899 @@
+
package eu.dnetlib.dhp.schema.oaf.utils;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.oaf.*;
-
-import eu.dnetlib.dhp.schema.oaf.common.AccessRightComparator;
-import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Pair;
+import static com.google.common.base.Preconditions.checkArgument;
+import static eu.dnetlib.dhp.schema.oaf.common.ModelSupport.isSubClass;
+import static eu.dnetlib.dhp.schema.oaf.common.ModelSupport.sameClass;
import java.text.ParseException;
import java.util.*;
import java.util.stream.Collectors;
-import static com.google.common.base.Preconditions.checkArgument;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.*;
+import eu.dnetlib.dhp.schema.oaf.common.AccessRightComparator;
+import eu.dnetlib.dhp.schema.oaf.common.ModelSupport;
public class MergeUtils {
- public static Oaf merge(final Oaf left, final Oaf right) {
- if (ModelSupport.isSubClass(left, Entity.class)) {
- return mergeEntities((Entity) left, (Entity) right);
- } else if (ModelSupport.isSubClass(left, Relation.class)) {
- return MergeUtils.mergeRelation((Relation) left, (Relation) right);
- } else {
- throw new IllegalArgumentException("invalid Oaf type:" + left.getClass().getCanonicalName());
- }
- }
-
- public static Entity mergeEntities(Entity original, Entity enrich) {
- if (ModelSupport.isSubClass(original, Result.class)) {
- return mergeResults((Result) original, (Result) enrich);
- } else if (ModelSupport.isSubClass(original, Datasource.class)) {
- //TODO
- return original;
- } else if (ModelSupport.isSubClass(original, Organization.class)) {
- return mergeOrganization((Organization) original, (Organization) enrich);
- } else if (ModelSupport.isSubClass(original, Project.class)) {
- return mergeProject((Project) original, (Project) enrich);
- } else {
- throw new IllegalArgumentException("invalid Entity subtype:" + original.getClass().getCanonicalName());
- }
- }
-
- public static Result mergeResults(Result original, Result enrich) {
-
- final boolean leftFromDelegatedAuthority = isFromDelegatedAuthority(original);
- final boolean rightFromDelegatedAuthority = isFromDelegatedAuthority(enrich);
-
- if (leftFromDelegatedAuthority && !rightFromDelegatedAuthority) {
- return original;
- }
- if (!leftFromDelegatedAuthority && rightFromDelegatedAuthority) {
- return enrich;
- }
- if (new ResultTypeComparator().compare(original, enrich) < 0) {
- return MergeUtils.mergeResult(original, enrich);
- } else {
- return MergeUtils.mergeResult(enrich, original);
- }
- }
-
- public static Result mergeResult(Result original, Result enrich) {
-
- final Result mergedResult = (Result) mergeEntity(original, enrich);
-
- if(StringUtils.isBlank(mergedResult.getProcessingchargeamount())){
- mergedResult.setProcessingchargeamount(enrich.getProcessingchargeamount());
- mergedResult.setProcessingchargecurrency(enrich.getProcessingchargecurrency());
- }
-
- mergedResult.setMeasures(mergeLists(mergedResult.getMeasures(), enrich.getMeasures()));
-
- if( !isAnEnrichment(mergedResult) && !isAnEnrichment(enrich))
- mergedResult.setInstance(mergeLists(mergedResult.getInstance(), enrich.getInstance()));
- else {
- final List enrichmentInstances = isAnEnrichment(mergedResult) ? mergedResult.getInstance() : enrich.getInstance();
- final List enrichedInstances= isAnEnrichment(mergedResult) ? enrich.getInstance(): mergedResult.getInstance();
- if (isAnEnrichment(mergedResult))
- mergedResult.setDataInfo(enrich.getDataInfo());
- mergedResult.setInstance(enrichInstances(enrichedInstances,enrichmentInstances));
- }
-
- if (enrich.getBestaccessright() != null
- && new AccessRightComparator<>().compare(enrich.getBestaccessright(), mergedResult.getBestaccessright()) < 0)
- mergedResult.setBestaccessright(enrich.getBestaccessright());
-
- final int trustCompareResult = compareTrust(mergedResult, enrich);
-
- if (enrich.getResulttype() != null && trustCompareResult < 0)
- mergedResult.setResulttype(enrich.getResulttype());
-
- if (enrich.getLanguage() != null && trustCompareResult < 0)
- mergedResult.setLanguage(enrich.getLanguage());
-
- if (Objects.nonNull(enrich.getDateofacceptance())) {
- if (Objects.isNull(mergedResult.getDateofacceptance())) {
- mergedResult.setDateofacceptance(enrich.getDateofacceptance());
- } else if (trustCompareResult < 0) {
- mergedResult.setDateofacceptance(enrich.getDateofacceptance());
- }
- }
+ public static T merge(final T left, final T right) {
+ if (sameClass(left, right, Entity.class)) {
+ return mergeEntities(left, right);
+ } else if (sameClass(left, right, Relation.class)) {
+ return mergeRelation(left, right);
+ } else {
+ throw new RuntimeException(
+ String
+ .format(
+ "MERGE_FROM_AND_GET incompatible types: %s, %s",
+ left.getClass().getCanonicalName(), right.getClass().getCanonicalName()));
+ }
+ }
+
+ private static T mergeEntities(T left, T right) {
+ if (sameClass(left, right, Result.class)) {
+ if (!left.getClass().equals(right.getClass())) {
+ return mergeResultsOfDifferentTypes(left, right);
+ }
+ return mergeResult(left, right);
+ } else if (sameClass(left, right, Datasource.class)) {
+ // TODO
+ return left;
+ } else if (sameClass(left, right, Organization.class)) {
+ return mergeOrganization(left, right);
+ } else if (sameClass(left, right, Project.class)) {
+ return mergeProject(left, right);
+ } else {
+ throw new RuntimeException(
+ String
+ .format(
+ "MERGE_FROM_AND_GET incompatible types: %s, %s",
+ left.getClass().getCanonicalName(), right.getClass().getCanonicalName()));
+ }
+ }
+
+ /**
+ * This method is used in the global result grouping phase. It checks if one of the two is from a delegated authority
+ * https://graph.openaire.eu/docs/data-model/pids-and-identifiers#delegated-authorities and in that case it prefers
+ * such version.
+ *
+ * Otherwise, it considers a resulttype priority order implemented in {@link ResultTypeComparator}
+ * and proceeds with the canonical property merging.
+ *
+ * @param left
+ * @param right
+ * @return
+ */
+ private static T mergeResultsOfDifferentTypes(T left, T right) {
+
+ final boolean leftFromDelegatedAuthority = isFromDelegatedAuthority((Result) left);
+ final boolean rightFromDelegatedAuthority = isFromDelegatedAuthority((Result) right);
+
+ if (leftFromDelegatedAuthority && !rightFromDelegatedAuthority) {
+ return left;
+ }
+ if (!leftFromDelegatedAuthority && rightFromDelegatedAuthority) {
+ return right;
+ }
+ if (new ResultTypeComparator().compare((Result) left, (Result) right) < 0) {
+ return mergeResult(left, right);
+ } else {
+ return mergeResult(right, left);
+ }
+ }
+
+ /**
+ * Internal utility that merges the common entity fields
+ *
+ * @param left
+ * @param right
+ * @return
+ * @param
+ */
+ private static T mergeEntityFields(T left, T right) {
+
+ final Entity enrich = (Entity) right;
+ final Entity mergedEntity = (Entity) left;
+
+ mergedEntity.setOriginalId(mergeLists(mergedEntity.getOriginalId(), enrich.getOriginalId()));
+ mergedEntity.setCollectedfrom(mergeLists(mergedEntity.getCollectedfrom(), enrich.getCollectedfrom()));
+
+ if (mergedEntity.getLastupdatetimestamp() == null && enrich.getLastupdatetimestamp() != null) {
+ mergedEntity.setLastupdatetimestamp(enrich.getLastupdatetimestamp());
+ } else if (mergedEntity.getLastupdatetimestamp() != null && enrich.getLastupdatetimestamp() != null) {
+ mergedEntity
+ .setLastupdatetimestamp(
+ Long.max(mergedEntity.getLastupdatetimestamp(), enrich.getLastupdatetimestamp()));
+ }
+
+ mergedEntity.setPid(mergeLists(mergedEntity.getPid(), enrich.getPid()));
+
+ final int trustCompareResult = compareTrust(mergedEntity, enrich);
+ if (enrich.getDateofcollection() != null && trustCompareResult < 0)
+ mergedEntity.setDateofcollection(enrich.getDateofcollection());
+
+ if (enrich.getDateoftransformation() != null && trustCompareResult < 0)
+ mergedEntity.setDateoftransformation(enrich.getDateoftransformation());
+
+ mergedEntity.setMeasures(mergeLists(mergedEntity.getMeasures(), enrich.getMeasures()));
+ mergedEntity.setExtraInfo(mergeLists(mergedEntity.getExtraInfo(), enrich.getExtraInfo()));
+
+ return (T) mergedEntity;
+ }
+
+ private static T mergeRelation(T left, T right) {
+
+ Relation original = (Relation) left;
+ Relation enrich = (Relation) right;
+
+ checkArgument(Objects.equals(original.getSource(), enrich.getSource()), "source ids must be equal");
+ checkArgument(Objects.equals(original.getTarget(), enrich.getTarget()), "target ids must be equal");
+ checkArgument(Objects.equals(original.getRelType(), enrich.getRelType()), "relType(s) must be equal");
+ checkArgument(
+ Objects.equals(original.getSubRelType(), enrich.getSubRelType()), "subRelType(s) must be equal");
+ checkArgument(Objects.equals(original.getRelClass(), enrich.getRelClass()), "relClass(es) must be equal");
+
+ original.setProvenance(mergeLists(original.getProvenance(), enrich.getProvenance()));
+
+ original.setValidated(original.getValidated() || enrich.getValidated());
+ try {
+ original.setValidationDate(ModelSupport.oldest(original.getValidationDate(), enrich.getValidationDate()));
+ } catch (ParseException e) {
+ throw new IllegalArgumentException(String
+ .format(
+ "invalid validation date format in relation [s:%s, t:%s]: %s", original.getSource(),
+ original.getTarget(),
+ original.getValidationDate()));
+ }
+
+ return (T) original;
+ }
+
+ private static T mergeResult(T left, T right) {
+
+ Result original = (Result) left;
+ Result enrich = (Result) right;
+
+ final Result mergedResult = mergeEntityFields(original, enrich);
+
+ if (StringUtils.isBlank(mergedResult.getProcessingchargeamount())) {
+ mergedResult.setProcessingchargeamount(enrich.getProcessingchargeamount());
+ mergedResult.setProcessingchargecurrency(enrich.getProcessingchargecurrency());
+ }
+
+ mergedResult.setMeasures(mergeLists(mergedResult.getMeasures(), enrich.getMeasures()));
+
+ if (!isAnEnrichment(mergedResult) && !isAnEnrichment(enrich))
+ mergedResult.setInstance(mergeLists(mergedResult.getInstance(), enrich.getInstance()));
+ else {
+ final List enrichmentInstances = isAnEnrichment(mergedResult) ? mergedResult.getInstance()
+ : enrich.getInstance();
+ final List enrichedInstances = isAnEnrichment(mergedResult) ? enrich.getInstance()
+ : mergedResult.getInstance();
+ if (isAnEnrichment(mergedResult))
+ mergedResult.setDataInfo(enrich.getDataInfo());
+ mergedResult.setInstance(enrichInstances(enrichedInstances, enrichmentInstances));
+ }
+
+ if (enrich.getBestaccessright() != null
+ && new AccessRightComparator<>()
+ .compare(enrich.getBestaccessright(), mergedResult.getBestaccessright()) < 0)
+ mergedResult.setBestaccessright(enrich.getBestaccessright());
- mergedResult.setCountry(mergeLists(mergedResult.getCountry(), enrich.getCountry()));
+ final int trustCompareResult = compareTrust(mergedResult, enrich);
- mergedResult.setSubject(mergeLists(mergedResult.getSubject(), enrich.getSubject()));
+ if (enrich.getResulttype() != null && trustCompareResult < 0)
+ mergedResult.setResulttype(enrich.getResulttype());
- if (enrich.getJournal() != null && trustCompareResult < 0)
- mergedResult.setJournal(enrich.getJournal());
+ if (enrich.getLanguage() != null && trustCompareResult < 0)
+ mergedResult.setLanguage(enrich.getLanguage());
- // merge title lists: main title with higher trust and distinct between the others
- StructuredProperty baseMainTitle = null;
- if (mergedResult.getTitle() != null) {
- baseMainTitle = getMainTitle(mergedResult.getTitle());
- if (baseMainTitle != null) {
- final StructuredProperty p = baseMainTitle;
- mergedResult.setTitle(mergedResult.getTitle().stream().filter(t -> t != p).collect(Collectors.toList()));
- }
- }
+ if (Objects.nonNull(enrich.getDateofacceptance())) {
+ if (Objects.isNull(mergedResult.getDateofacceptance()) || trustCompareResult < 0) {
+ mergedResult.setDateofacceptance(enrich.getDateofacceptance());
+ }
+ }
- StructuredProperty newMainTitle = null;
- if (enrich.getTitle() != null) {
- newMainTitle = getMainTitle(enrich.getTitle());
- if (newMainTitle != null) {
- final StructuredProperty p = newMainTitle;
- enrich.setTitle(enrich.getTitle().stream().filter(t -> t != p).collect(Collectors.toList()));
- }
- }
+ mergedResult.setCountry(mergeLists(mergedResult.getCountry(), enrich.getCountry()));
- if (newMainTitle != null && trustCompareResult < 0) {
- baseMainTitle = newMainTitle;
- }
+ mergedResult.setSubject(mergeLists(mergedResult.getSubject(), enrich.getSubject()));
- mergedResult.setTitle(mergeLists(mergedResult.getTitle(), enrich.getTitle()));
- if (mergedResult.getTitle() != null && baseMainTitle != null) {
- mergedResult.getTitle().add(baseMainTitle);
- }
+ if (enrich.getJournal() != null && trustCompareResult < 0)
+ mergedResult.setJournal(enrich.getJournal());
- mergedResult.setRelevantdate(mergeLists(mergedResult.getRelevantdate(), enrich.getRelevantdate()));
+ // merge title lists: main title with higher trust and distinct between the others
+ StructuredProperty baseMainTitle = null;
+ if (mergedResult.getTitle() != null) {
+ baseMainTitle = getMainTitle(mergedResult.getTitle());
+ if (baseMainTitle != null) {
+ final StructuredProperty p = baseMainTitle;
+ mergedResult
+ .setTitle(mergedResult.getTitle().stream().filter(t -> t != p).collect(Collectors.toList()));
+ }
+ }
- mergedResult.setDescription(longestLists(mergedResult.getDescription(), enrich.getDescription()));
+ StructuredProperty newMainTitle = null;
+ if (enrich.getTitle() != null) {
+ newMainTitle = getMainTitle(enrich.getTitle());
+ if (newMainTitle != null) {
+ final StructuredProperty p = newMainTitle;
+ enrich.setTitle(enrich.getTitle().stream().filter(t -> t != p).collect(Collectors.toList()));
+ }
+ }
- if (enrich.getPublisher() != null && trustCompareResult < 0)
- mergedResult.setPublisher(enrich.getPublisher());
+ if (newMainTitle != null && trustCompareResult < 0) {
+ baseMainTitle = newMainTitle;
+ }
- if (enrich.getEmbargoenddate() != null && trustCompareResult < 0)
- mergedResult.setEmbargoenddate(enrich.getEmbargoenddate());
+ mergedResult.setTitle(mergeLists(mergedResult.getTitle(), enrich.getTitle()));
+ if (mergedResult.getTitle() != null && baseMainTitle != null) {
+ mergedResult.getTitle().add(baseMainTitle);
+ }
- mergedResult.setSource(mergeLists(mergedResult.getSource(), enrich.getSource()));
+ mergedResult.setRelevantdate(mergeLists(mergedResult.getRelevantdate(), enrich.getRelevantdate()));
- mergedResult.setFulltext(mergeLists(mergedResult.getFulltext(), enrich.getFulltext()));
+ mergedResult.setDescription(longestLists(mergedResult.getDescription(), enrich.getDescription()));
- mergedResult.setFormat(mergeLists(mergedResult.getFormat(), enrich.getFormat()));
+ if (enrich.getPublisher() != null && trustCompareResult < 0)
+ mergedResult.setPublisher(enrich.getPublisher());
- mergedResult.setContributor(mergeLists(mergedResult.getContributor(), enrich.getContributor()));
+ if (enrich.getEmbargoenddate() != null && trustCompareResult < 0)
+ mergedResult.setEmbargoenddate(enrich.getEmbargoenddate());
- if (enrich.getResourcetype() != null)
- mergedResult.setResourcetype(enrich.getResourcetype());
+ mergedResult.setSource(mergeLists(mergedResult.getSource(), enrich.getSource()));
- mergedResult.setCoverage(mergeLists(mergedResult.getCoverage(), enrich.getCoverage()));
+ mergedResult.setFulltext(mergeLists(mergedResult.getFulltext(), enrich.getFulltext()));
- mergedResult.setContext(mergeLists(mergedResult.getContext(), enrich.getContext()));
+ mergedResult.setFormat(mergeLists(mergedResult.getFormat(), enrich.getFormat()));
- mergedResult.setExternalReference(mergeLists(mergedResult.getExternalReference(), enrich.getExternalReference()));
+ mergedResult.setContributor(mergeLists(mergedResult.getContributor(), enrich.getContributor()));
- if (enrich.getOaiprovenance() != null && trustCompareResult < 0)
- mergedResult.setOaiprovenance(enrich.getOaiprovenance());
+ if (enrich.getResourcetype() != null)
+ mergedResult.setResourcetype(enrich.getResourcetype());
- return mergedResult;
- }
+ mergedResult.setCoverage(mergeLists(mergedResult.getCoverage(), enrich.getCoverage()));
- public static OtherResearchProduct mergeORP(OtherResearchProduct original, OtherResearchProduct enrich) {
- final OtherResearchProduct mergedORP = (OtherResearchProduct) mergeResult(original, enrich);
+ mergedResult.setContext(mergeLists(mergedResult.getContext(), enrich.getContext()));
- mergedORP.setContactperson(mergeLists(mergedORP.getContactperson(), enrich.getContactperson()));
- mergedORP.setContactgroup(mergeLists(mergedORP.getContactgroup(), enrich.getContactgroup()));
- mergedORP.setTool(mergeLists(mergedORP.getTool(), enrich.getTool()));
- mergeEntityDataInfo(mergedORP, enrich);
+ mergedResult
+ .setExternalReference(mergeLists(mergedResult.getExternalReference(), enrich.getExternalReference()));
- return mergedORP;
- }
+ if (enrich.getOaiprovenance() != null && trustCompareResult < 0)
+ mergedResult.setOaiprovenance(enrich.getOaiprovenance());
- public static Software mergeSoftware(Software original, Software enrich) {
- final Software mergedSoftware = (Software) mergeResult(original, enrich);
+ if (isSubClass(mergedResult, Publication.class)) {
+ return (T) mergePublication(mergedResult, enrich);
+ }
+ if (isSubClass(mergedResult, Dataset.class)) {
+ return (T) mergeDataset(mergedResult, enrich);
+ }
+ if (isSubClass(mergedResult, OtherResearchProduct.class)) {
+ return (T) mergeORP(mergedResult, enrich);
+ }
+ if (isSubClass(mergedResult, Software.class)) {
+ return (T) mergeSoftware(mergedResult, enrich);
+ }
- mergedSoftware.setDocumentationUrl(mergeLists(mergedSoftware.getDocumentationUrl(), enrich.getDocumentationUrl()));
+ mergeEntityDataInfo(original, enrich);
- mergedSoftware.setCodeRepositoryUrl(enrich.getCodeRepositoryUrl() != null && compareTrust(mergedSoftware,enrich) < 0
- ? enrich.getCodeRepositoryUrl()
- : mergedSoftware.getCodeRepositoryUrl());
+ return (T) mergedResult;
+ }
- mergedSoftware.setProgrammingLanguage(enrich.getProgrammingLanguage() != null && compareTrust(mergedSoftware, enrich) < 0
- ? enrich.getProgrammingLanguage()
- : mergedSoftware.getProgrammingLanguage());
+ private static T mergeORP(T left, T right) {
- mergeEntityDataInfo(mergedSoftware, enrich);
- return mergedSoftware;
- }
+ final OtherResearchProduct original = (OtherResearchProduct) left;
+ final OtherResearchProduct enrich = (OtherResearchProduct) right;
- public static Dataset mergeDataset(Dataset original, Dataset enrich) {
+ original.setContactperson(mergeLists(original.getContactperson(), enrich.getContactperson()));
+ original.setContactgroup(mergeLists(original.getContactgroup(), enrich.getContactgroup()));
+ original.setTool(mergeLists(original.getTool(), enrich.getTool()));
- final Dataset mergedDataset = (Dataset) mergeResult(original, enrich);
+ mergeEntityDataInfo(original, enrich);
- mergedDataset.setStoragedate(enrich.getStoragedate() != null && compareTrust(mergedDataset, enrich) < 0 ? enrich.getStoragedate() : mergedDataset.getStoragedate());
+ return (T) original;
+ }
- mergedDataset.setDevice(enrich.getDevice() != null && compareTrust(mergedDataset, enrich) < 0 ? enrich.getDevice() : mergedDataset.getDevice());
+ private static T mergeSoftware(T left, T right) {
+ final Software original = (Software) left;
+ final Software enrich = (Software) right;
- mergedDataset.setSize(enrich.getSize() != null && compareTrust(mergedDataset, enrich) < 0 ? enrich.getSize() : mergedDataset.getSize());
+ original
+ .setDocumentationUrl(mergeLists(original.getDocumentationUrl(), enrich.getDocumentationUrl()));
- mergedDataset.setVersion(enrich.getVersion() != null && compareTrust(mergedDataset, enrich) < 0 ? enrich.getVersion() : mergedDataset.getVersion());
+ original
+ .setCodeRepositoryUrl(
+ enrich.getCodeRepositoryUrl() != null && compareTrust(original, enrich) < 0
+ ? enrich.getCodeRepositoryUrl()
+ : original.getCodeRepositoryUrl());
- mergedDataset.setLastmetadataupdate(
- enrich.getLastmetadataupdate() != null && compareTrust(mergedDataset,enrich) < 0
- ? enrich.getLastmetadataupdate()
- : mergedDataset.getLastmetadataupdate());
+ original
+ .setProgrammingLanguage(
+ enrich.getProgrammingLanguage() != null && compareTrust(original, enrich) < 0
+ ? enrich.getProgrammingLanguage()
+ : original.getProgrammingLanguage());
- mergedDataset.setMetadataversionnumber(
- enrich.getMetadataversionnumber() != null && compareTrust(mergedDataset, enrich) < 0
- ? enrich.getMetadataversionnumber()
- : mergedDataset.getMetadataversionnumber());
+ mergeEntityDataInfo(original, enrich);
- mergedDataset.setGeolocation(mergeLists(mergedDataset.getGeolocation(), enrich.getGeolocation()));
+ return (T) original;
+ }
- mergeEntityDataInfo(mergedDataset, enrich);
+ private static T mergeDataset(T left, T right) {
+ Dataset original = (Dataset) left;
+ Dataset enrich = (Dataset) right;
- return mergedDataset;
- }
+ original
+ .setStoragedate(
+ enrich.getStoragedate() != null && compareTrust(original, enrich) < 0 ? enrich.getStoragedate()
+ : original.getStoragedate());
- public static Publication mergePublication(Publication original, Publication enrich) {
+ original
+ .setDevice(
+ enrich.getDevice() != null && compareTrust(original, enrich) < 0 ? enrich.getDevice()
+ : original.getDevice());
- final Publication mergedPublication = (Publication) mergeResult(original, enrich);
+ original
+ .setSize(
+ enrich.getSize() != null && compareTrust(original, enrich) < 0 ? enrich.getSize()
+ : original.getSize());
- mergeEntityDataInfo(mergedPublication, enrich);
- return mergedPublication;
- }
+ original
+ .setVersion(
+ enrich.getVersion() != null && compareTrust(original, enrich) < 0 ? enrich.getVersion()
+ : original.getVersion());
- public static Organization mergeOrganization(Organization original, Organization enrich) {
+ original
+ .setLastmetadataupdate(
+ enrich.getLastmetadataupdate() != null && compareTrust(original, enrich) < 0
+ ? enrich.getLastmetadataupdate()
+ : original.getLastmetadataupdate());
+
+ original
+ .setMetadataversionnumber(
+ enrich.getMetadataversionnumber() != null && compareTrust(original, enrich) < 0
+ ? enrich.getMetadataversionnumber()
+ : original.getMetadataversionnumber());
+
+ original.setGeolocation(mergeLists(original.getGeolocation(), enrich.getGeolocation()));
+
+ mergeEntityDataInfo(original, enrich);
+
+ return (T) original;
+ }
+
+ private static T mergePublication(T original, T enrich) {
+
+ //add publication specific fields.
+
+ mergeEntityDataInfo(original, enrich);
+
+ return original;
+ }
+
+ private static T mergeOrganization(T left, T right) {
+
+ Organization original = (Organization) left;
+ Organization enrich = (Organization) right;
+
+ final Organization mergedOrganization = mergeEntityFields(original, enrich);
+
+ int ct = compareTrust(mergedOrganization, enrich);
+ mergedOrganization
+ .setLegalshortname(
+ enrich.getLegalshortname() != null && ct < 0
+ ? enrich.getLegalshortname()
+ : mergedOrganization.getLegalname());
+
+ mergedOrganization
+ .setLegalname(
+ enrich.getLegalname() != null && ct < 0 ? enrich.getLegalname()
+ : mergedOrganization.getLegalname());
+
+ mergedOrganization
+ .setAlternativeNames(mergeLists(enrich.getAlternativeNames(), mergedOrganization.getAlternativeNames()));
+
+ mergedOrganization
+ .setWebsiteurl(
+ enrich.getWebsiteurl() != null && ct < 0
+ ? enrich.getWebsiteurl()
+ : mergedOrganization.getWebsiteurl());
+
+ mergedOrganization
+ .setLogourl(
+ enrich.getLogourl() != null && ct < 0
+ ? enrich.getLogourl()
+ : mergedOrganization.getLogourl());
+
+ mergedOrganization
+ .setEclegalbody(
+ enrich.getEclegalbody() != null && ct < 0
+ ? enrich.getEclegalbody()
+ : mergedOrganization.getEclegalbody());
+
+ mergedOrganization
+ .setEclegalperson(
+ enrich.getEclegalperson() != null && ct < 0
+ ? enrich.getEclegalperson()
+ : mergedOrganization.getEclegalperson());
+
+ mergedOrganization
+ .setEcnonprofit(
+ enrich.getEcnonprofit() != null && ct < 0
+ ? enrich.getEcnonprofit()
+ : mergedOrganization.getEcnonprofit());
+
+ mergedOrganization
+ .setEcresearchorganization(
+ enrich.getEcresearchorganization() != null && ct < 0
+ ? enrich.getEcresearchorganization()
+ : mergedOrganization.getEcresearchorganization());
+
+ mergedOrganization
+ .setEchighereducation(
+ enrich.getEchighereducation() != null && ct < 0
+ ? enrich.getEchighereducation()
+ : mergedOrganization.getEchighereducation());
+
+ mergedOrganization
+ .setEcinternationalorganizationeurinterests(
+ enrich.getEcinternationalorganizationeurinterests() != null && ct < 0
+ ? enrich.getEcinternationalorganizationeurinterests()
+ : mergedOrganization.getEcinternationalorganizationeurinterests());
+
+ mergedOrganization
+ .setEcinternationalorganization(
+ enrich.getEcinternationalorganization() != null && ct < 0
+ ? enrich.getEcinternationalorganization()
+ : mergedOrganization.getEcinternationalorganization());
+
+ mergedOrganization
+ .setEcenterprise(
+ enrich.getEcenterprise() != null && ct < 0
+ ? enrich.getEcenterprise()
+ : mergedOrganization.getEcenterprise());
+
+ mergedOrganization
+ .setEcsmevalidated(
+ enrich.getEcsmevalidated() != null && ct < 0
+ ? enrich.getEcsmevalidated()
+ : mergedOrganization.getEcsmevalidated());
+ mergedOrganization
+ .setEcnutscode(
+ enrich.getEcnutscode() != null && ct < 0
+ ? enrich.getEcnutscode()
+ : mergedOrganization.getEcnutscode());
+
+ mergedOrganization
+ .setCountry(
+ enrich.getCountry() != null && ct < 0 ? enrich.getCountry()
+ : mergedOrganization.getCountry());
+
+ mergeEntityDataInfo(mergedOrganization, enrich);
+
+ return (T) mergedOrganization;
+ }
+
+ public static T mergeProject(T left, T right) {
+
+ Project original = (Project) left;
+ Project enrich = (Project) right;
+
+ final Project mergedProject = mergeEntityFields(original, enrich);
+
+ int ct = compareTrust(mergedProject, enrich);
+
+ mergedProject
+ .setWebsiteurl(
+ enrich.getWebsiteurl() != null && ct < 0
+ ? enrich.getWebsiteurl()
+ : mergedProject.getWebsiteurl());
+
+ mergedProject.setCode(enrich.getCode() != null && ct < 0 ? enrich.getCode() : mergedProject.getCode());
+
+ mergedProject
+ .setAcronym(
+ enrich.getAcronym() != null && ct < 0
+ ? enrich.getAcronym()
+ : mergedProject.getAcronym());
+
+ mergedProject
+ .setTitle(
+ enrich.getTitle() != null && ct < 0
+ ? enrich.getTitle()
+ : mergedProject.getTitle());
+ mergedProject
+ .setStartdate(
+ enrich.getStartdate() != null && ct < 0
+ ? enrich.getStartdate()
+ : mergedProject.getStartdate());
+ mergedProject
+ .setEnddate(
+ enrich.getEnddate() != null && ct < 0
+ ? enrich.getEnddate()
+ : mergedProject.getEnddate());
+ mergedProject
+ .setCallidentifier(
+ enrich.getCallidentifier() != null && ct < 0
+ ? enrich.getCallidentifier()
+ : mergedProject.getCallidentifier());
+ mergedProject
+ .setKeywords(
+ enrich.getKeywords() != null && ct < 0
+ ? enrich.getKeywords()
+ : mergedProject.getKeywords());
+
+ mergedProject
+ .setDuration(
+ enrich.getDuration() != null && ct < 0
+ ? enrich.getDuration()
+ : mergedProject.getDuration());
+ mergedProject
+ .setEcsc39(
+ enrich.getEcsc39() != null && ct < 0
+ ? enrich.getEcsc39()
+ : mergedProject.getEcsc39());
+ mergedProject
+ .setOamandatepublications(
+ enrich.getOamandatepublications() != null && ct < 0
+ ? enrich.getOamandatepublications()
+ : mergedProject.getOamandatepublications());
+ mergedProject
+ .setEcarticle29_3(
+ enrich.getEcarticle29_3() != null && ct < 0
+ ? enrich.getEcarticle29_3()
+ : mergedProject.getEcarticle29_3());
+
+ mergedProject.setSubjects(mergeLists(mergedProject.getSubjects(), enrich.getSubjects()));
+ mergedProject.setFundingtree(mergeLists(mergedProject.getFundingtree(), enrich.getFundingtree()));
+ mergedProject
+ .setContracttype(
+ enrich.getContracttype() != null && ct < 0
+ ? enrich.getContracttype()
+ : mergedProject.getContracttype());
+ mergedProject
+ .setOptional1(
+ enrich.getOptional1() != null && ct < 0
+ ? enrich.getOptional1()
+ : mergedProject.getOptional1());
+ mergedProject
+ .setOptional2(
+ enrich.getOptional2() != null && ct < 0
+ ? enrich.getOptional2()
+ : mergedProject.getOptional2());
+
+ mergedProject
+ .setJsonextrainfo(
+ enrich.getJsonextrainfo() != null && ct < 0
+ ? enrich.getJsonextrainfo()
+ : mergedProject.getJsonextrainfo());
+
+ mergedProject
+ .setContactfullname(
+ enrich.getContactfullname() != null && ct < 0
+ ? enrich.getContactfullname()
+ : mergedProject.getContactfullname());
+
+ mergedProject
+ .setContactfax(
+ enrich.getContactfax() != null && ct < 0
+ ? enrich.getContactfax()
+ : mergedProject.getContactfax());
+
+ mergedProject
+ .setContactphone(
+ enrich.getContactphone() != null && ct < 0
+ ? enrich.getContactphone()
+ : mergedProject.getContactphone());
+
+ mergedProject
+ .setContactemail(
+ enrich.getContactemail() != null && ct < 0
+ ? enrich.getContactemail()
+ : mergedProject.getContactemail());
+
+ mergedProject
+ .setSummary(
+ enrich.getSummary() != null && ct < 0
+ ? enrich.getSummary()
+ : mergedProject.getSummary());
+
+ mergedProject
+ .setCurrency(
+ enrich.getCurrency() != null && ct < 0
+ ? enrich.getCurrency()
+ : mergedProject.getCurrency());
+
+ if (enrich.getH2020topiccode() != null && StringUtils.isEmpty(mergedProject.getH2020topiccode())) {
+ mergedProject.setH2020topiccode(enrich.getH2020topiccode());
+ mergedProject.setH2020topicdescription(enrich.getH2020topicdescription());
+ }
+
+ mergedProject
+ .setH2020classification(
+ mergeLists(mergedProject.getH2020classification(), enrich.getH2020classification()));
+
+ mergeEntityDataInfo(mergedProject, enrich);
+
+ return (T) mergedProject;
+ }
+
+ private static void mergeEntityDataInfo(T left, T right) {
+ Entity l = (Entity) left;
+ Entity r = (Entity) right;
+ Optional
+ .ofNullable(r)
+ .ifPresent(
+ other -> Optional
+ .ofNullable(other.getDataInfo())
+ .ifPresent(
+ otherDataInfo -> Optional
+ .ofNullable(l.getDataInfo())
+ .ifPresent(thisDataInfo -> {
+ if (compareTrust(r, other) < 0 || thisDataInfo.getInvisible()) {
+ l.setDataInfo(otherDataInfo);
+ }
+ })));
+ }
+
+ /**
+ * Gets main title.
+ *
+ * @param titles the titles
+ * @return the main title
+ */
+ private static StructuredProperty getMainTitle(List titles) {
+ // need to check if the list of titles contains more than 1 main title? (in that case, we should chose which
+ // main title select in the list)
+ for (StructuredProperty t : titles) {
+ if (t.getQualifier() != null && t.getQualifier().getClassid() != null)
+ if (t.getQualifier().getClassid().equals("main title"))
+ return t;
+ }
+ return null;
+ }
+
+ /**
+ * Longest lists list.
+ *
+ * @param a the a
+ * @param b the b
+ * @return the list
+ */
+ public static List longestLists(List a, List b) {
+ if (a == null || b == null)
+ return a == null ? b : a;
+ if (a.size() == b.size()) {
+ int msa = a
+ .stream()
+ .filter(i -> i != null)
+ .map(i -> i.length())
+ .max(Comparator.naturalOrder())
+ .orElse(0);
+ int msb = b
+ .stream()
+ .filter(i -> i != null)
+ .map(i -> i.length())
+ .max(Comparator.naturalOrder())
+ .orElse(0);
+ return msa > msb ? a : b;
+ }
+ return a.size() > b.size() ? a : b;
+ }
+
+ /**
+ * This main method apply the enrichment of the instances
+ *
+ * @param toEnrichInstances the instances that could be enriched
+ * @param enrichmentInstances the enrichment instances
+ * @return list of instances possibly enriched
+ */
+ private static List enrichInstances(final List toEnrichInstances,
+ final List enrichmentInstances) {
+ final List enrichmentResult = new ArrayList<>();
+
+ if (toEnrichInstances == null) {
+ return enrichmentResult;
+ }
+ if (enrichmentInstances == null) {
+ return enrichmentResult;
+ }
+ Map ri = toInstanceMap(enrichmentInstances);
+
+ toEnrichInstances.forEach(i -> {
+ final List e = findEnrichmentsByPID(i.getPid(), ri);
+ if (e != null && e.size() > 0) {
+ e.forEach(enr -> applyEnrichment(i, enr));
+ } else {
+ final List a = findEnrichmentsByPID(i.getAlternateIdentifier(), ri);
+ if (a != null && a.size() > 0) {
+ a.forEach(enr -> applyEnrichment(i, enr));
+ }
+ }
+ enrichmentResult.add(i);
+ });
+ return enrichmentResult;
+ }
+
+ /**
+ * This method converts the list of instance enrichments
+ * into a Map where the key is the normalized identifier
+ * and the value is the instance itself
+ *
+ * @param ri the list of enrichment instances
+ * @return the result map
+ */
+ private static Map toInstanceMap(final List ri) {
+ return ri
+ .stream()
+ .filter(i -> i.getPid() != null || i.getAlternateIdentifier() != null)
+ .flatMap(i -> {
+ final List> result = new ArrayList<>();
+ if (i.getPid() != null)
+ i
+ .getPid()
+ .stream()
+ .filter(MergeUtils::validPid)
+ .forEach(p -> result.add(new ImmutablePair<>(extractKeyFromPid(p), i)));
+ if (i.getAlternateIdentifier() != null)
+ i
+ .getAlternateIdentifier()
+ .stream()
+ .filter(MergeUtils::validPid)
+ .forEach(p -> result.add(new ImmutablePair<>(extractKeyFromPid(p), i)));
+ return result.stream();
+ })
+ .collect(
+ Collectors
+ .toMap(
+ Pair::getLeft,
+ Pair::getRight,
+ (a, b) -> a));
+ }
+
+ private static boolean isFromDelegatedAuthority(Result r) {
+ return Optional
+ .ofNullable(r.getInstance())
+ .map(
+ instance -> instance
+ .stream()
+ .filter(i -> Objects.nonNull(i.getCollectedfrom()))
+ .map(i -> i.getCollectedfrom().getKey())
+ .anyMatch(cfId -> IdentifierFactory.delegatedAuthorityDatasourceIds().contains(cfId)))
+ .orElse(false);
+ }
+
+ /**
+ * Valid pid boolean.
+ *
+ * @param p the p
+ * @return the boolean
+ */
+ private static boolean validPid(final StructuredProperty p) {
+ return p.getValue() != null && p.getQualifier() != null && p.getQualifier().getClassid() != null;
+ }
+
+ /**
+ * Normalize pid string.
+ *
+ * @param pid the pid
+ * @return the string
+ */
+ private static String extractKeyFromPid(final StructuredProperty pid) {
+ if (pid == null)
+ return null;
+ final StructuredProperty normalizedPid = CleaningFunctions.normalizePidValue(pid);
+
+ return String.format("%s::%s", normalizedPid.getQualifier().getClassid(), normalizedPid.getValue());
+ }
+
+ /**
+ * This utility method finds the list of enrichment instances
+ * that match one or more PIDs in the input list
+ *
+ * @param pids the list of PIDs
+ * @param enrichments the List of enrichment instances having the same pid
+ * @return the list
+ */
+ private static List findEnrichmentsByPID(final List pids,
+ final Map enrichments) {
+ if (pids == null || enrichments == null)
+ return null;
+ return pids
+ .stream()
+ .map(MergeUtils::extractKeyFromPid)
+ .map(enrichments::get)
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Is an enrichment boolean.
+ *
+ * @param e the e
+ * @return the boolean
+ */
+ public static boolean isAnEnrichment(Entity e) {
+ return e.getDataInfo() != null &&
+ e.getDataInfo().getProvenanceaction() != null
+ && ModelConstants.PROVENANCE_ENRICH.equalsIgnoreCase(e.getDataInfo().getProvenanceaction().getClassid());
+ }
+
+ /**
+ * This method apply enrichment on a single instance
+ * The enrichment consists of replacing values on
+ * single attribute only if in the current instance is missing
+ * The only repeatable field enriched is measures
+ *
+ * @param currentInstance the current instance
+ * @param enrichment the enrichment instance
+ */
+ private static void applyEnrichment(final Instance currentInstance, final Instance enrichment) {
+ if (currentInstance == null || enrichment == null)
+ return;
+
+ // ENRICH accessright
+ if (enrichment.getAccessright() != null && currentInstance.getAccessright() == null)
+ currentInstance.setAccessright(enrichment.getAccessright());
+
+ // ENRICH license
+ if (enrichment.getLicense() != null && currentInstance.getLicense() == null)
+ currentInstance.setLicense(enrichment.getLicense());
+
+ // ENRICH instanceType
+ if (enrichment.getInstancetype() != null && currentInstance.getInstancetype() == null)
+ currentInstance.setInstancetype(enrichment.getInstancetype());
+
+ // ENRICH hostedby
+ if (enrichment.getHostedby() != null && currentInstance.getHostedby() == null)
+ currentInstance.setHostedby(enrichment.getHostedby());
+
+ // ENRICH distributionlocation
+ if (enrichment.getDistributionlocation() != null && currentInstance.getDistributionlocation() == null)
+ currentInstance.setDistributionlocation(enrichment.getDistributionlocation());
+
+ // ENRICH collectedfrom
+ if (enrichment.getCollectedfrom() != null && currentInstance.getCollectedfrom() == null)
+ currentInstance.setCollectedfrom(enrichment.getCollectedfrom());
+
+ // ENRICH dateofacceptance
+ if (enrichment.getDateofacceptance() != null && currentInstance.getDateofacceptance() == null)
+ currentInstance.setDateofacceptance(enrichment.getDateofacceptance());
+
+ // ENRICH processingchargeamount
+ if (enrichment.getProcessingchargeamount() != null && currentInstance.getProcessingchargeamount() == null)
+ currentInstance.setProcessingchargeamount(enrichment.getProcessingchargeamount());
+
+ // ENRICH refereed
+ if (enrichment.getRefereed() != null && currentInstance.getRefereed() == null)
+ currentInstance.setRefereed(enrichment.getRefereed());
+
+ // TODO check the other Instance fields
+ }
+
+ private static List mergeLists(final List... lists) {
+ return Arrays
+ .stream(lists)
+ .filter(Objects::nonNull)
+ .flatMap(List::stream)
+ .filter(Objects::nonNull)
+ .distinct()
+ .collect(Collectors.toList());
+ }
+
+ private static int compareTrust(Entity a, Entity b) {
+ return Float
+ .compare(
+ Optional
+ .ofNullable(a.getDataInfo())
+ .map(DataInfo::getTrust)
+ .orElse(0f),
+ Optional
+ .ofNullable(b.getDataInfo())
+ .map(DataInfo::getTrust)
+ .orElse(0f));
+ }
- final Organization mergedOrganization = (Organization) mergeEntity(original, enrich);
-
- int ct = compareTrust(mergedOrganization, enrich);
- mergedOrganization.setLegalshortname(enrich.getLegalshortname() != null && ct < 0
- ? enrich.getLegalshortname()
- : mergedOrganization.getLegalname());
-
-
- mergedOrganization.setLegalname(enrich.getLegalname() != null && ct < 0 ?
- enrich.getLegalname()
- : mergedOrganization.getLegalname());
-
- mergedOrganization.setAlternativeNames(mergeLists(enrich.getAlternativeNames(), mergedOrganization.getAlternativeNames()));
-
-
- mergedOrganization.setWebsiteurl(enrich.getWebsiteurl() != null && ct < 0
- ? enrich.getWebsiteurl()
- : mergedOrganization.getWebsiteurl());
-
- mergedOrganization.setLogourl(enrich.getLogourl() != null && ct < 0
- ? enrich.getLogourl()
- : mergedOrganization.getLogourl());
-
- mergedOrganization.setEclegalbody(enrich.getEclegalbody() != null && ct < 0
- ? enrich.getEclegalbody()
- : mergedOrganization.getEclegalbody());
-
- mergedOrganization.setEclegalperson(enrich.getEclegalperson() != null && ct < 0
- ? enrich.getEclegalperson()
- : mergedOrganization.getEclegalperson());
-
- mergedOrganization.setEcnonprofit (enrich.getEcnonprofit() != null && ct< 0
- ? enrich.getEcnonprofit()
- : mergedOrganization.getEcnonprofit());
-
- mergedOrganization.setEcresearchorganization (enrich.getEcresearchorganization() != null && ct < 0
- ? enrich.getEcresearchorganization()
- : mergedOrganization.getEcresearchorganization());
-
- mergedOrganization.setEchighereducation (enrich.getEchighereducation() != null && ct < 0
- ? enrich.getEchighereducation()
- : mergedOrganization.getEchighereducation());
-
- mergedOrganization.setEcinternationalorganizationeurinterests (enrich.getEcinternationalorganizationeurinterests() != null && ct< 0
- ? enrich.getEcinternationalorganizationeurinterests()
- : mergedOrganization.getEcinternationalorganizationeurinterests());
-
- mergedOrganization.setEcinternationalorganization (enrich.getEcinternationalorganization() != null && ct < 0
- ? enrich.getEcinternationalorganization()
- : mergedOrganization.getEcinternationalorganization());
-
- mergedOrganization.setEcenterprise (enrich.getEcenterprise() != null && ct < 0
- ? enrich.getEcenterprise()
- : mergedOrganization.getEcenterprise());
-
- mergedOrganization.setEcsmevalidated (enrich.getEcsmevalidated() != null && ct < 0
- ? enrich.getEcsmevalidated()
- : mergedOrganization.getEcsmevalidated());
- mergedOrganization.setEcnutscode( enrich.getEcnutscode() != null && ct < 0
- ? enrich.getEcnutscode()
- : mergedOrganization.getEcnutscode());
-
- mergedOrganization.setCountry (enrich.getCountry() != null && ct < 0 ?
- enrich.getCountry()
- :mergedOrganization.getCountry());
-
- mergeEntityDataInfo(mergedOrganization, enrich);
-
- return mergedOrganization;
- }
-
- public static Project mergeProject(Project original, Project enrich) {
-
- final Project mergedProject = (Project) mergeEntity(original, enrich);
-
- int ct = compareTrust(mergedProject, enrich);
-
-
- mergedProject.setWebsiteurl (enrich.getWebsiteurl() != null && ct < 0
- ? enrich.getWebsiteurl()
- : mergedProject.getWebsiteurl());
-
- mergedProject.setCode(enrich.getCode() != null && ct < 0 ?
- enrich.getCode() :
- mergedProject.getCode());
-
- mergedProject.setAcronym(enrich.getAcronym() != null && ct < 0
- ? enrich.getAcronym()
- : mergedProject.getAcronym());
-
- mergedProject.setTitle (enrich.getTitle() != null && ct < 0
- ? enrich.getTitle()
- : mergedProject.getTitle());
- mergedProject.setStartdate (enrich.getStartdate() != null && ct < 0
- ? enrich.getStartdate()
- : mergedProject.getStartdate());
- mergedProject.setEnddate (enrich.getEnddate() != null && ct < 0
- ? enrich.getEnddate()
- : mergedProject.getEnddate());
- mergedProject.setCallidentifier ( enrich.getCallidentifier() != null && ct < 0
- ? enrich.getCallidentifier()
- : mergedProject.getCallidentifier());
- mergedProject.setKeywords ( enrich.getKeywords() != null && ct < 0
- ? enrich.getKeywords()
- : mergedProject.getKeywords());
-
- mergedProject.setDuration ( enrich.getDuration() != null && ct < 0
- ? enrich.getDuration()
- : mergedProject.getDuration());
- mergedProject.setEcsc39 ( enrich.getEcsc39() != null && ct < 0
- ? enrich.getEcsc39() :
- mergedProject.getEcsc39());
- mergedProject.setOamandatepublications ( enrich.getOamandatepublications() != null && ct < 0
- ? enrich.getOamandatepublications()
- : mergedProject.getOamandatepublications());
- mergedProject.setEcarticle29_3 (enrich.getEcarticle29_3() != null && ct < 0
- ? enrich.getEcarticle29_3()
- : mergedProject.getEcarticle29_3());
-
- mergedProject.setSubjects (mergeLists(mergedProject.getSubjects(), enrich.getSubjects()));
- mergedProject.setFundingtree (mergeLists(mergedProject.getFundingtree(), enrich.getFundingtree()));
- mergedProject.setContracttype (enrich.getContracttype() != null && ct < 0
- ? enrich.getContracttype()
- : mergedProject.getContracttype());
- mergedProject.setOptional1 ( enrich.getOptional1() != null && ct < 0
- ? enrich.getOptional1()
- : mergedProject.getOptional1());
- mergedProject.setOptional2 (enrich.getOptional2() != null && ct < 0
- ? enrich.getOptional2()
- : mergedProject.getOptional2());
-
- mergedProject.setJsonextrainfo ( enrich.getJsonextrainfo() != null && ct < 0
- ? enrich.getJsonextrainfo()
- : mergedProject.getJsonextrainfo());
-
- mergedProject.setContactfullname ( enrich.getContactfullname() != null && ct < 0
- ? enrich.getContactfullname()
- : mergedProject.getContactfullname());
-
- mergedProject.setContactfax ( enrich.getContactfax() != null && ct < 0
- ? enrich.getContactfax()
- : mergedProject.getContactfax());
-
- mergedProject.setContactphone (enrich.getContactphone() != null && ct < 0
- ? enrich.getContactphone()
- : mergedProject.getContactphone());
-
- mergedProject.setContactemail ( enrich.getContactemail() != null && ct < 0
- ? enrich.getContactemail()
- : mergedProject.getContactemail());
-
- mergedProject.setSummary ( enrich.getSummary() != null && ct < 0
- ? enrich.getSummary()
- : mergedProject.getSummary());
-
- mergedProject.setCurrency( enrich.getCurrency() != null && ct < 0
- ? enrich.getCurrency()
- : mergedProject.getCurrency());
-
- if (enrich.getH2020topiccode() != null && StringUtils.isEmpty(mergedProject.getH2020topiccode())){
- mergedProject.setH2020topiccode(enrich.getH2020topiccode());
- mergedProject.setH2020topicdescription(enrich.getH2020topicdescription());
- }
-
- mergedProject.setH2020classification(mergeLists(mergedProject.getH2020classification(), enrich.getH2020classification()));
-
- mergeEntityDataInfo(mergedProject, enrich);
-
- return mergedProject;
- }
-
- public static Entity mergeEntity(Entity original, Entity enrich) {
-
- final Entity mergedEntity = original;
-
- mergedEntity.setOriginalId(mergeLists(mergedEntity.getOriginalId(), enrich.getOriginalId()));
- mergedEntity.setCollectedfrom(mergeLists(mergedEntity.getCollectedfrom(), enrich.getCollectedfrom()));
-
- if (mergedEntity.getLastupdatetimestamp() == null && enrich.getLastupdatetimestamp() != null) {
- mergedEntity.setLastupdatetimestamp(enrich.getLastupdatetimestamp());
- } else if (mergedEntity.getLastupdatetimestamp() != null && enrich.getLastupdatetimestamp() != null) {
- mergedEntity.setLastupdatetimestamp(Long.max(mergedEntity.getLastupdatetimestamp(), enrich.getLastupdatetimestamp()));
- }
-
- mergedEntity.setPid(mergeLists(mergedEntity.getPid(), enrich.getPid()));
-
- final int trustCompareResult = compareTrust(mergedEntity, enrich);
- if (enrich.getDateofcollection() != null && trustCompareResult < 0)
- mergedEntity.setDateofcollection(enrich.getDateofcollection());
-
- if (enrich.getDateoftransformation() != null && trustCompareResult < 0)
- mergedEntity.setDateoftransformation(enrich.getDateoftransformation());
-
- mergedEntity.setMeasures(mergeLists(mergedEntity.getMeasures(), enrich.getMeasures()));
- mergedEntity.setExtraInfo(mergeLists(mergedEntity.getExtraInfo(), enrich.getExtraInfo()));
-
- return mergedEntity;
- }
-
- public static Relation mergeRelation(Relation original, Relation enrich) {
-
- checkArgument(Objects.equals(original.getSource(), enrich.getSource()), "source ids must be equal");
- checkArgument(Objects.equals(original.getTarget(), enrich.getTarget()), "target ids must be equal");
- checkArgument(Objects.equals(original.getRelType(), enrich.getRelType()), "relType(s) must be equal");
- checkArgument(
- Objects.equals(original.getSubRelType(), enrich.getSubRelType()), "subRelType(s) must be equal");
- checkArgument(Objects.equals(original.getRelClass(), enrich.getRelClass()), "relClass(es) must be equal");
-
- original.setProvenance(mergeLists(original.getProvenance(), enrich.getProvenance()));
-
- original.setValidated(original.getValidated() || enrich.getValidated());
- try {
- original.setValidationDate(ModelSupport.oldest(original.getValidationDate(), enrich.getValidationDate()));
- } catch (ParseException e) {
- throw new IllegalArgumentException(String
- .format(
- "invalid validation date format in relation [s:%s, t:%s]: %s", original.getSource(), original.getTarget(),
- original.getValidationDate()));
- }
-
- return original;
- }
-
- private static void mergeEntityDataInfo(Entity from, Entity to) {
- Optional.ofNullable(to)
- .ifPresent(other -> Optional.ofNullable(other.getDataInfo())
- .ifPresent(otherDataInfo -> Optional.ofNullable(from.getDataInfo())
- .ifPresent(thisDataInfo -> {
- if (compareTrust(from, other) < 0 || thisDataInfo.getInvisible()) {
- from.setDataInfo(otherDataInfo);
- }
- })));
- }
-
- /**
- * Gets main title.
- *
- * @param titles the titles
- * @return the main title
- */
- private static StructuredProperty getMainTitle(List titles) {
- // need to check if the list of titles contains more than 1 main title? (in that case, we should chose which
- // main title select in the list)
- for (StructuredProperty t : titles) {
- if (t.getQualifier() != null && t.getQualifier().getClassid() != null)
- if (t.getQualifier().getClassid().equals("main title"))
- return t;
- }
- return null;
- }
-
- /**
- * Longest lists list.
- *
- * @param a the a
- * @param b the b
- * @return the list
- */
- public static List longestLists(List a, List b) {
- if (a == null || b == null)
- return a == null ? b : a;
- if (a.size() == b.size()) {
- int msa = a
- .stream()
- .filter(i -> i != null)
- .map(i -> i.length())
- .max(Comparator.naturalOrder())
- .orElse(0);
- int msb = b
- .stream()
- .filter(i -> i != null )
- .map(i -> i.length())
- .max(Comparator.naturalOrder())
- .orElse(0);
- return msa > msb ? a : b;
- }
- return a.size() > b.size() ? a : b;
- }
-
- /**
- * This main method apply the enrichment of the instances
- *
- * @param toEnrichInstances the instances that could be enriched
- * @param enrichmentInstances the enrichment instances
- * @return list of instances possibly enriched
- */
- private static List enrichInstances(final List toEnrichInstances,final List enrichmentInstances) {
- final List enrichmentResult = new ArrayList<>();
-
- if (toEnrichInstances == null) {
- return enrichmentResult;
- }
- if (enrichmentInstances == null) {
- return enrichmentResult;
- }
- Map ri = toInstanceMap(enrichmentInstances);
-
- toEnrichInstances.forEach(i -> {
- final List e = findEnrichmentsByPID(i.getPid(), ri);
- if (e!= null && e.size()> 0) {
- e.forEach(enr -> applyEnrichment(i, enr));
- } else {
- final List a = findEnrichmentsByPID(i.getAlternateIdentifier(), ri);
- if (a!= null && a.size()> 0) {
- a.forEach(enr -> applyEnrichment(i, enr));
- }
- }
- enrichmentResult.add(i);
- });
- return enrichmentResult;
- }
-
- /**
- * This method converts the list of instance enrichments
- * into a Map where the key is the normalized identifier
- * and the value is the instance itself
- *
- * @param ri the list of enrichment instances
- * @return the result map
- */
- private static Map toInstanceMap(final List ri) {
- return ri
- .stream()
- .filter(i -> i.getPid() != null || i.getAlternateIdentifier() != null)
- .flatMap(i -> {
- final List> result = new ArrayList<>();
- if (i.getPid() != null)
- i.getPid().stream().filter(MergeUtils::validPid).forEach(p -> result.add(new ImmutablePair<>(extractKeyFromPid(p), i)));
- if (i.getAlternateIdentifier() != null)
- i.getAlternateIdentifier().stream().filter(MergeUtils::validPid).forEach(p -> result.add(new ImmutablePair<>(extractKeyFromPid(p), i)));
- return result.stream();
- }).collect(Collectors.toMap(
- Pair::getLeft,
- Pair::getRight,
- (a, b) -> a
- ));
- }
-
- private static boolean isFromDelegatedAuthority(Result r) {
- return Optional
- .ofNullable(r.getInstance())
- .map(
- instance -> instance
- .stream()
- .filter(i -> Objects.nonNull(i.getCollectedfrom()))
- .map(i -> i.getCollectedfrom().getKey())
- .anyMatch(cfId -> IdentifierFactory.delegatedAuthorityDatasourceIds().contains(cfId)))
- .orElse(false);
- }
-
- /**
- * Valid pid boolean.
- *
- * @param p the p
- * @return the boolean
- */
- private static boolean validPid(final StructuredProperty p) {
- return p.getValue()!= null && p.getQualifier()!= null && p.getQualifier().getClassid()!=null;
- }
-
- /**
- * Normalize pid string.
- *
- * @param pid the pid
- * @return the string
- */
- private static String extractKeyFromPid(final StructuredProperty pid) {
- if (pid == null)
- return null;
- final StructuredProperty normalizedPid = CleaningFunctions.normalizePidValue(pid);
-
- return String.format("%s::%s", normalizedPid.getQualifier().getClassid(), normalizedPid.getValue());
- }
-
- /**
- * This utility method finds the list of enrichment instances
- * that match one or more PIDs in the input list
- *
- * @param pids the list of PIDs
- * @param enrichments the List of enrichment instances having the same pid
- * @return the list
- */
- private static List findEnrichmentsByPID(final List pids, final Map enrichments) {
- if (pids == null || enrichments == null)
- return null;
- return pids
- .stream()
- .map(MergeUtils::extractKeyFromPid)
- .map(enrichments::get)
- .filter(Objects::nonNull)
- .collect(Collectors.toList());
- }
-
- /**
- * Is an enrichment boolean.
- *
- * @param e the e
- * @return the boolean
- */
- public static boolean isAnEnrichment(Entity e) {
- return e.getDataInfo() != null &&
- e.getDataInfo().getProvenanceaction()!= null
- && ModelConstants.PROVENANCE_ENRICH.equalsIgnoreCase(e.getDataInfo().getProvenanceaction().getClassid());
- }
-
- /**
- * This method apply enrichment on a single instance
- * The enrichment consists of replacing values on
- * single attribute only if in the current instance is missing
- * The only repeatable field enriched is measures
- *
- * @param currentInstance the current instance
- * @param enrichment the enrichment instance
- */
- private static void applyEnrichment(final Instance currentInstance, final Instance enrichment) {
- if (currentInstance == null || enrichment == null)
- return;
-
- //ENRICH accessright
- if (enrichment.getAccessright()!=null && currentInstance.getAccessright() == null)
- currentInstance.setAccessright(enrichment.getAccessright());
-
- //ENRICH license
- if (enrichment.getLicense()!=null && currentInstance.getLicense() == null)
- currentInstance.setLicense(enrichment.getLicense());
-
- //ENRICH instanceType
- if (enrichment.getInstancetype()!=null && currentInstance.getInstancetype() == null)
- currentInstance.setInstancetype(enrichment.getInstancetype());
-
- //ENRICH hostedby
- if (enrichment.getHostedby()!=null && currentInstance.getHostedby() == null)
- currentInstance.setHostedby(enrichment.getHostedby());
-
- //ENRICH distributionlocation
- if (enrichment.getDistributionlocation()!=null && currentInstance.getDistributionlocation() == null)
- currentInstance.setDistributionlocation(enrichment.getDistributionlocation());
-
- //ENRICH collectedfrom
- if (enrichment.getCollectedfrom()!=null && currentInstance.getCollectedfrom() == null)
- currentInstance.setCollectedfrom(enrichment.getCollectedfrom());
-
- //ENRICH dateofacceptance
- if (enrichment.getDateofacceptance()!=null && currentInstance.getDateofacceptance() == null)
- currentInstance.setDateofacceptance(enrichment.getDateofacceptance());
-
- //ENRICH processingchargeamount
- if (enrichment.getProcessingchargeamount()!=null && currentInstance.getProcessingchargeamount() == null)
- currentInstance.setProcessingchargeamount(enrichment.getProcessingchargeamount());
-
- //ENRICH refereed
- if (enrichment.getRefereed()!=null && currentInstance.getRefereed() == null)
- currentInstance.setRefereed(enrichment.getRefereed());
-
- //TODO check the other Instance fields
- }
-
- private static List mergeLists(final List... lists) {
- return Arrays
- .stream(lists)
- .filter(Objects::nonNull)
- .flatMap(List::stream)
- .filter(Objects::nonNull)
- .distinct()
- .collect(Collectors.toList());
- }
-
- private static int compareTrust(Entity a, Entity b) {
- return Float.compare(
- Optional.ofNullable(a.getDataInfo())
- .map(DataInfo::getTrust)
- .orElse(0f),
- Optional.ofNullable(b.getDataInfo())
- .map(DataInfo::getTrust)
- .orElse(0f));
- }
-
}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils2.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils2.java
deleted file mode 100644
index 60ea5bf1f..000000000
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils2.java
+++ /dev/null
@@ -1,156 +0,0 @@
-package eu.dnetlib.dhp.schema.oaf.utils;
-
-import java.lang.reflect.Field;
-import java.util.Collection;
-import java.util.Iterator;
-
-public class MergeUtils2 {
-
- /**
- * Recursively merges the fields of the provider into the receiver.
- *
- * @param receiver the receiver instance.
- * @param provider the provider instance.
- */
- public static void merge(final T receiver, final T provider) {
- Field[] fields = receiver.getClass().getDeclaredFields();
- for (Field field : fields) {
-
- try {
- field.setAccessible(true);
- Object receiverObject = field.get(receiver);
- Object providerObject = field.get(provider);
-
- if (receiverObject == null || providerObject == null) {
- /* One is null */
-
- field.set(receiver, providerObject);
- } else if (field.getType().isAssignableFrom(Collection.class)) {
- /* Collection field */
- // noinspection rawtypes
- mergeCollections((Collection) receiverObject, (Collection) providerObject);
- } else if (field.getType().isPrimitive() || field.getType().isEnum()
- || field.getType().equals(String.class)) {
- /* Primitive, Enum or String field */
- field.set(receiver, providerObject);
- } else {
- /* Mergeable field */
- merge(receiverObject, providerObject);
- }
- } catch (IllegalAccessException e) {
- /* Should not happen */
- throw new RuntimeException(e);
- }
- }
- }
-
- /**
- * Recursively merges the items in the providers collection into the receivers collection.
- * Receivers not present in providers will be removed, providers not present in receivers will be added.
- * If the item has a field called 'id', this field will be compared to match the items.
- *
- * @param receivers the collection containing the receiver instances.
- * @param providers the collection containing the provider instances.
- */
- public static void mergeCollections(final Collection receivers, final Collection providers) {
- if (receivers.isEmpty() && providers.isEmpty()) {
- return;
- }
-
- if (providers.isEmpty()) {
- receivers.clear();
- return;
- }
-
- if (receivers.isEmpty()) {
- receivers.addAll(providers);
- return;
- }
-
- Field idField;
- try {
- T t = providers.iterator().next();
- idField = t.getClass().getDeclaredField("id");
- idField.setAccessible(true);
- } catch (NoSuchFieldException ignored) {
- idField = null;
- }
-
- try {
- if (idField != null) {
- mergeCollectionsWithId(receivers, providers, idField);
- } else {
- mergeCollectionsSimple(receivers, providers);
- }
- } catch (IllegalAccessException e) {
- /* Should not happen */
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Recursively merges the items in the collections for which the id's are equal.
- *
- * @param receivers the collection containing the receiver items.
- * @param providers the collection containing the provider items.
- * @param idField the id field.
- *
- * @throws IllegalAccessException if the id field is not accessible.
- */
- private static void mergeCollectionsWithId(final Collection receivers, final Iterable providers,
- final Field idField) throws IllegalAccessException {
- /* Find a receiver for each provider */
- for (T provider : providers) {
- boolean found = false;
- for (T receiver : receivers) {
- if (idField.get(receiver).equals(idField.get(provider))) {
- merge(receiver, provider);
- found = true;
- }
- }
- if (!found) {
- receivers.add(provider);
- }
- }
-
- /* Remove receivers not in providers */
- for (Iterator iterator = receivers.iterator(); iterator.hasNext();) {
- T receiver = iterator.next();
- boolean found = false;
- for (T provider : providers) {
- if (idField.get(receiver).equals(idField.get(provider))) {
- found = true;
- }
- }
- if (!found) {
- iterator.remove();
- }
- }
- }
-
- /**
- * Recursively merges the items in the collections one by one. Disregards equality.
- *
- * @param receivers the collection containing the receiver items.
- * @param providers the collection containing the provider items.
- */
- private static void mergeCollectionsSimple(final Collection receivers, final Iterable providers) {
- Iterator receiversIterator = receivers.iterator();
- Iterator providersIterator = providers.iterator();
- while (receiversIterator.hasNext() && providersIterator.hasNext()) {
- merge(receiversIterator.next(), providersIterator.next());
- }
-
- /* Remove excessive receivers if present */
- while (receiversIterator.hasNext()) {
- receiversIterator.next();
- receiversIterator.remove();
- }
-
- /* Add residual providers to receivers if present */
- while (providersIterator.hasNext()) {
- receivers.add(providersIterator.next());
- }
- }
-
-}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils3.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils3.java
deleted file mode 100644
index cb3f67c8b..000000000
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils3.java
+++ /dev/null
@@ -1,89 +0,0 @@
-package eu.dnetlib.dhp.schema.oaf.utils;
-
-import java.lang.reflect.Field;
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-
-import static org.apache.commons.lang3.ClassUtils.isPrimitiveOrWrapper;
-
-public class MergeUtils3 {
-
- private final List