report) {
+ sendMessage(new Message(MessageType.REPORT, workflowId, report));
+ }
+
+ private Message createOngoingMessage(final Long current, final Long total) {
+ final Message m = new Message(MessageType.ONGOING, workflowId);
+ m.getBody().put(Message.CURRENT_PARAM, current.toString());
+ if (total != null) {
+ m.getBody().put(Message.TOTAL_PARAM, total.toString());
+ }
+ return m;
+ }
+
+ private void _sendMessage(final Message message) {
+ try {
+ final String json = objectMapper.writeValueAsString(message);
+
+ final HttpPut req = new HttpPut(dnetMessageEndpoint);
+ req.setEntity(new StringEntity(json, ContentType.APPLICATION_JSON));
+
+ final RequestConfig requestConfig = RequestConfig
+ .custom()
+ .setConnectTimeout(CONNTECTION_TIMEOUT_MS)
+ .setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MS)
+ .setSocketTimeout(SOCKET_TIMEOUT_MS)
+ .build();
+
+ try (final CloseableHttpClient client = HttpClients
+ .custom()
+ .setDefaultRequestConfig(requestConfig)
+ .build();
+ final CloseableHttpResponse response = client.execute(req)) {
+ log.debug("Sent Message to " + dnetMessageEndpoint);
+ log.debug("MESSAGE:" + message);
+ } catch (final Throwable e) {
+ log.error("Error sending message to " + dnetMessageEndpoint + ", message content: " + message, e);
+ }
+ } catch (final JsonProcessingException e) {
+ log.error("Error sending message to " + dnetMessageEndpoint + ", message content: " + message, e);
+ }
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/message/MessageType.java b/dhp-common/src/main/java/eu/dnetlib/dhp/message/MessageType.java
new file mode 100644
index 000000000..75ffb8ef5
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/message/MessageType.java
@@ -0,0 +1,21 @@
+
+package eu.dnetlib.dhp.message;
+
+import java.io.Serializable;
+import java.util.Optional;
+
+import org.apache.commons.lang3.StringUtils;
+
+public enum MessageType implements Serializable {
+
+ ONGOING, REPORT;
+
+ public MessageType from(String value) {
+ return Optional
+ .ofNullable(value)
+ .map(StringUtils::upperCase)
+ .map(MessageType::valueOf)
+ .orElseThrow(() -> new IllegalArgumentException("unknown message type: " + value));
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/model/mdstore/MetadataRecord.java b/dhp-common/src/main/java/eu/dnetlib/dhp/model/mdstore/MetadataRecord.java
deleted file mode 100644
index ce65e710f..000000000
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/model/mdstore/MetadataRecord.java
+++ /dev/null
@@ -1,121 +0,0 @@
-
-package eu.dnetlib.dhp.model.mdstore;
-
-import java.io.Serializable;
-
-import eu.dnetlib.dhp.utils.DHPUtils;
-
-/** This class models a record inside the new Metadata store collection on HDFS * */
-public class MetadataRecord implements Serializable {
-
- /** The D-Net Identifier associated to the record */
- private String id;
-
- /** The original Identifier of the record */
- private String originalId;
-
- /** The encoding of the record, should be JSON or XML */
- private String encoding;
-
- /**
- * The information about the provenance of the record see @{@link Provenance} for the model of this information
- */
- private Provenance provenance;
-
- /** The content of the metadata */
- private String body;
-
- /** the date when the record has been stored */
- private long dateOfCollection;
-
- /** the date when the record has been stored */
- private long dateOfTransformation;
-
- public MetadataRecord() {
- this.dateOfCollection = System.currentTimeMillis();
- }
-
- public MetadataRecord(
- String originalId,
- String encoding,
- Provenance provenance,
- String body,
- long dateOfCollection) {
-
- this.originalId = originalId;
- this.encoding = encoding;
- this.provenance = provenance;
- this.body = body;
- this.dateOfCollection = dateOfCollection;
- this.id = DHPUtils.generateIdentifier(originalId, this.provenance.getNsPrefix());
- }
-
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- public String getOriginalId() {
- return originalId;
- }
-
- public void setOriginalId(String originalId) {
- this.originalId = originalId;
- }
-
- public String getEncoding() {
- return encoding;
- }
-
- public void setEncoding(String encoding) {
- this.encoding = encoding;
- }
-
- public Provenance getProvenance() {
- return provenance;
- }
-
- public void setProvenance(Provenance provenance) {
- this.provenance = provenance;
- }
-
- public String getBody() {
- return body;
- }
-
- public void setBody(String body) {
- this.body = body;
- }
-
- public long getDateOfCollection() {
- return dateOfCollection;
- }
-
- public void setDateOfCollection(long dateOfCollection) {
- this.dateOfCollection = dateOfCollection;
- }
-
- public long getDateOfTransformation() {
- return dateOfTransformation;
- }
-
- public void setDateOfTransformation(long dateOfTransformation) {
- this.dateOfTransformation = dateOfTransformation;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof MetadataRecord)) {
- return false;
- }
- return ((MetadataRecord) o).getId().equalsIgnoreCase(id);
- }
-
- @Override
- public int hashCode() {
- return id.hashCode();
- }
-}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/model/mdstore/Provenance.java b/dhp-common/src/main/java/eu/dnetlib/dhp/model/mdstore/Provenance.java
deleted file mode 100644
index 556535022..000000000
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/model/mdstore/Provenance.java
+++ /dev/null
@@ -1,52 +0,0 @@
-
-package eu.dnetlib.dhp.model.mdstore;
-
-import java.io.Serializable;
-
-/**
- * @author Sandro La Bruzzo
- *
- * Provenace class models the provenance of the record in the metadataStore It contains the identifier and the
- * name of the datasource that gives the record
- */
-public class Provenance implements Serializable {
-
- private String datasourceId;
-
- private String datasourceName;
-
- private String nsPrefix;
-
- public Provenance() {
- }
-
- public Provenance(String datasourceId, String datasourceName, String nsPrefix) {
- this.datasourceId = datasourceId;
- this.datasourceName = datasourceName;
- this.nsPrefix = nsPrefix;
- }
-
- public String getDatasourceId() {
- return datasourceId;
- }
-
- public void setDatasourceId(String datasourceId) {
- this.datasourceId = datasourceId;
- }
-
- public String getDatasourceName() {
- return datasourceName;
- }
-
- public void setDatasourceName(String datasourceName) {
- this.datasourceName = datasourceName;
- }
-
- public String getNsPrefix() {
- return nsPrefix;
- }
-
- public void setNsPrefix(String nsPrefix) {
- this.nsPrefix = nsPrefix;
- }
-}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java
index 17482c019..0461c9353 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java
@@ -18,6 +18,9 @@ public class AuthorMerger {
private static final Double THRESHOLD = 0.95;
+ private AuthorMerger() {
+ }
+
public static List merge(List> authors) {
authors.sort((o1, o2) -> -Integer.compare(countAuthorsPids(o1), countAuthorsPids(o2)));
@@ -32,44 +35,54 @@ public class AuthorMerger {
}
- public static List mergeAuthor(final List a, final List b) {
+ public static List mergeAuthor(final List a, final List b, Double threshold) {
int pa = countAuthorsPids(a);
int pb = countAuthorsPids(b);
- List base, enrich;
+ List base;
+ List enrich;
int sa = authorsSize(a);
int sb = authorsSize(b);
- if (pa == pb) {
- base = sa > sb ? a : b;
- enrich = sa > sb ? b : a;
- } else {
+ if (sa == sb) {
base = pa > pb ? a : b;
enrich = pa > pb ? b : a;
+ } else {
+ base = sa > sb ? a : b;
+ enrich = sa > sb ? b : a;
}
- enrichPidFromList(base, enrich);
+ enrichPidFromList(base, enrich, threshold);
return base;
}
- private static void enrichPidFromList(List base, List enrich) {
+ public static List mergeAuthor(final List a, final List b) {
+ return mergeAuthor(a, b, THRESHOLD);
+ }
+
+ private static void enrichPidFromList(List base, List enrich, Double threshold) {
if (base == null || enrich == null)
return;
+
+ // (if an Author has more than 1 pid, it appears 2 times in the list)
final Map basePidAuthorMap = base
.stream()
- .filter(a -> a.getPid() != null && a.getPid().size() > 0)
+ .filter(a -> a.getPid() != null && !a.getPid().isEmpty())
.flatMap(
a -> a
.getPid()
.stream()
+ .filter(Objects::nonNull)
.map(p -> new Tuple2<>(pidToComparableString(p), a)))
.collect(Collectors.toMap(Tuple2::_1, Tuple2::_2, (x1, x2) -> x1));
+ // (list of pid that are missing in the other list)
final List> pidToEnrich = enrich
.stream()
- .filter(a -> a.getPid() != null && a.getPid().size() > 0)
+ .filter(a -> a.getPid() != null && !a.getPid().isEmpty())
.flatMap(
a -> a
.getPid()
.stream()
+ .filter(Objects::nonNull)
.filter(p -> !basePidAuthorMap.containsKey(pidToComparableString(p)))
.map(p -> new Tuple2<>(p, a)))
.collect(Collectors.toList());
@@ -83,10 +96,10 @@ public class AuthorMerger {
.max(Comparator.comparing(Tuple2::_1));
if (simAuthor.isPresent()) {
- double th = THRESHOLD;
+ double th = threshold;
// increase the threshold if the surname is too short
if (simAuthor.get()._2().getSurname() != null
- && simAuthor.get()._2().getSurname().length() <= 3)
+ && simAuthor.get()._2().getSurname().length() <= 3 && threshold > 0.0)
th = 0.99;
if (simAuthor.get()._1() > th) {
@@ -106,10 +119,135 @@ public class AuthorMerger {
});
}
+ public static String normalizeFullName(final String fullname) {
+ return nfd(fullname)
+ .toLowerCase()
+ // do not compact the regexes in a single expression, would cause StackOverflowError
+ // in case
+ // of large input strings
+ .replaceAll("(\\W)+", " ")
+ .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ")
+ .replaceAll("(\\p{Punct})+", " ")
+ .replaceAll("(\\d)+", " ")
+ .replaceAll("(\\n)+", " ")
+
+ .trim();
+ }
+
+ private static String authorFieldToBeCompared(Author author) {
+ if (StringUtils.isNotBlank(author.getSurname())) {
+ return author.getSurname();
+
+ }
+ if (StringUtils.isNotBlank(author.getFullname())) {
+ return author.getFullname();
+ }
+ return null;
+ }
+
+ /**
+ * This method tries to figure out when two author are the same in the contest
+ * of ORCID enrichment
+ *
+ * @param left Author in the OAF entity
+ * @param right Author ORCID
+ * @return based on a heuristic on the names of the authors if they are the same.
+ */
+ public static boolean checkORCIDSimilarity(final Author left, final Author right) {
+ final Person pl = parse(left);
+ final Person pr = parse(right);
+
+ // If one of them didn't have a surname we verify if they have the fullName not empty
+ // and verify if the normalized version is equal
+ if (!(pl.getSurname() != null && pl.getSurname().stream().anyMatch(StringUtils::isNotBlank) &&
+ pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) {
+
+ if (pl.getFullname() != null && !pl.getFullname().isEmpty() && pr.getFullname() != null
+ && !pr.getFullname().isEmpty()) {
+ return pl
+ .getFullname()
+ .stream()
+ .anyMatch(
+ fl -> pr.getFullname().stream().anyMatch(fr -> normalize(fl).equalsIgnoreCase(normalize(fr))));
+ } else {
+ return false;
+ }
+ }
+ // The Authors have one surname in common
+ if (pl.getSurname().stream().anyMatch(sl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(sl)))) {
+
+ // If one of them has only a surname and is the same we can say that they are the same author
+ if ((pl.getName() == null || pl.getName().stream().allMatch(StringUtils::isBlank)) ||
+ (pr.getName() == null || pr.getName().stream().allMatch(StringUtils::isBlank)))
+ return true;
+ // The authors have the same initials of Name in common
+ if (pl
+ .getName()
+ .stream()
+ .anyMatch(
+ nl -> pr
+ .getName()
+ .stream()
+ .anyMatch(nr -> nr.equalsIgnoreCase(nl))))
+ return true;
+ }
+
+ // Sometimes we noticed that publication have author wrote in inverse order Surname, Name
+ // We verify if we have an exact match between name and surname
+ if (pl.getSurname().stream().anyMatch(sl -> pr.getName().stream().anyMatch(nr -> nr.equalsIgnoreCase(sl))) &&
+ pl.getName().stream().anyMatch(nl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(nl))))
+ return true;
+ else
+ return false;
+ }
+ //
+
+ /**
+ * Method to enrich ORCID information in one list of authors based on another list
+ *
+ * @param baseAuthor the Author List in the OAF Entity
+ * @param orcidAuthor The list of ORCID Author intersected
+ * @return The Author List of the OAF Entity enriched with the orcid Author
+ */
+ public static List enrichOrcid(List baseAuthor, List orcidAuthor) {
+
+ if (baseAuthor == null || baseAuthor.isEmpty())
+ return orcidAuthor;
+
+ if (orcidAuthor == null || orcidAuthor.isEmpty())
+ return baseAuthor;
+
+ if (baseAuthor.size() == 1 && orcidAuthor.size() > 10)
+ return baseAuthor;
+
+ final List oAuthor = new ArrayList<>();
+ oAuthor.addAll(orcidAuthor);
+
+ baseAuthor.forEach(ba -> {
+ Optional aMatch = oAuthor.stream().filter(oa -> checkORCIDSimilarity(ba, oa)).findFirst();
+ if (aMatch.isPresent()) {
+ final Author sameAuthor = aMatch.get();
+ addPid(ba, sameAuthor.getPid());
+ oAuthor.remove(sameAuthor);
+ }
+ });
+ return baseAuthor;
+ }
+
+ private static void addPid(final Author a, final List pids) {
+
+ if (a.getPid() == null) {
+ a.setPid(new ArrayList<>());
+ }
+
+ a.getPid().addAll(pids);
+
+ }
+
public static String pidToComparableString(StructuredProperty pid) {
- return (pid.getQualifier() != null
- ? pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() : ""
- : "")
+ final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase()
+ : "";
+ return (pid.getQualifier() != null ? classid : "")
+ (pid.getValue() != null ? pid.getValue().toLowerCase() : "");
}
@@ -142,7 +280,7 @@ public class AuthorMerger {
}
private static boolean hasPid(Author a) {
- if (a == null || a.getPid() == null || a.getPid().size() == 0)
+ if (a == null || a.getPid() == null || a.getPid().isEmpty())
return false;
return a.getPid().stream().anyMatch(p -> p != null && StringUtils.isNotBlank(p.getValue()));
}
@@ -151,12 +289,15 @@ public class AuthorMerger {
if (StringUtils.isNotBlank(author.getSurname())) {
return new Person(author.getSurname() + ", " + author.getName(), false);
} else {
- return new Person(author.getFullname(), false);
+ if (StringUtils.isNotBlank(author.getFullname()))
+ return new Person(author.getFullname(), false);
+ else
+ return new Person("", false);
}
}
- private static String normalize(final String s) {
- return nfd(s)
+ public static String normalize(final String s) {
+ String[] normalized = nfd(s)
.toLowerCase()
// do not compact the regexes in a single expression, would cause StackOverflowError
// in case
@@ -166,7 +307,12 @@ public class AuthorMerger {
.replaceAll("(\\p{Punct})+", " ")
.replaceAll("(\\d)+", " ")
.replaceAll("(\\n)+", " ")
- .trim();
+ .trim()
+ .split(" ");
+
+ Arrays.sort(normalized);
+
+ return String.join(" ", normalized);
}
private static String nfd(final String s) {
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java
new file mode 100644
index 000000000..0225a5063
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java
@@ -0,0 +1,194 @@
+
+package eu.dnetlib.dhp.oa.merge;
+
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+import static org.apache.spark.sql.functions.col;
+import static org.apache.spark.sql.functions.when;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.api.java.function.ReduceFunction;
+import org.apache.spark.sql.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.HdfsSupport;
+import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
+import eu.dnetlib.dhp.schema.common.EntityType;
+import eu.dnetlib.dhp.schema.common.ModelSupport;
+import eu.dnetlib.dhp.schema.oaf.OafEntity;
+import eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions;
+import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
+import eu.dnetlib.dhp.utils.ISLookupClientFactory;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import scala.Tuple2;
+
+/**
+ * Groups the graph content by entity identifier to ensure ID uniqueness
+ */
+public class GroupEntitiesSparkJob {
+ private static final Logger log = LoggerFactory.getLogger(GroupEntitiesSparkJob.class);
+
+ private static final Encoder OAFENTITY_KRYO_ENC = Encoders.kryo(OafEntity.class);
+
+ private ArgumentApplicationParser parser;
+
+ public GroupEntitiesSparkJob(ArgumentApplicationParser parser) {
+ this.parser = parser;
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ String jsonConfiguration = IOUtils
+ .toString(
+ GroupEntitiesSparkJob.class
+ .getResourceAsStream(
+ "/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json"));
+ final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+ parser.parseArgument(args);
+
+ Boolean isSparkSessionManaged = Optional
+ .ofNullable(parser.get("isSparkSessionManaged"))
+ .map(Boolean::valueOf)
+ .orElse(Boolean.TRUE);
+ log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+ final String isLookupUrl = parser.get("isLookupUrl");
+ log.info("isLookupUrl: {}", isLookupUrl);
+
+ final ISLookUpService isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl);
+
+ new GroupEntitiesSparkJob(parser).run(isSparkSessionManaged, isLookupService);
+ }
+
+ public void run(Boolean isSparkSessionManaged, ISLookUpService isLookUpService)
+ throws ISLookUpException {
+
+ String graphInputPath = parser.get("graphInputPath");
+ log.info("graphInputPath: {}", graphInputPath);
+
+ String checkpointPath = parser.get("checkpointPath");
+ log.info("checkpointPath: {}", checkpointPath);
+
+ String outputPath = parser.get("outputPath");
+ log.info("outputPath: {}", outputPath);
+
+ boolean filterInvisible = Boolean.parseBoolean(parser.get("filterInvisible"));
+ log.info("filterInvisible: {}", filterInvisible);
+
+ SparkConf conf = new SparkConf();
+ conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ conf.registerKryoClasses(ModelSupport.getOafModelClasses());
+
+ final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookUpService);
+
+ runWithSparkSession(
+ conf,
+ isSparkSessionManaged,
+ spark -> {
+ HdfsSupport.remove(checkpointPath, spark.sparkContext().hadoopConfiguration());
+ groupEntities(spark, graphInputPath, checkpointPath, outputPath, filterInvisible, vocs);
+ });
+ }
+
+ private static void groupEntities(
+ SparkSession spark,
+ String inputPath,
+ String checkpointPath,
+ String outputPath,
+ boolean filterInvisible, VocabularyGroup vocs) {
+
+ Dataset allEntities = spark.emptyDataset(OAFENTITY_KRYO_ENC);
+
+ for (Map.Entry e : ModelSupport.entityTypes.entrySet()) {
+ String entity = e.getKey().name();
+ Class entityClass = e.getValue();
+ String entityInputPath = inputPath + "/" + entity;
+
+ if (!HdfsSupport.exists(entityInputPath, spark.sparkContext().hadoopConfiguration())) {
+ continue;
+ }
+
+ allEntities = allEntities
+ .union(
+ ((Dataset) spark
+ .read()
+ .schema(Encoders.bean(entityClass).schema())
+ .json(entityInputPath)
+ .filter("length(id) > 0")
+ .as(Encoders.bean(entityClass)))
+ .map((MapFunction) r -> r, OAFENTITY_KRYO_ENC));
+ }
+
+ Dataset groupedEntities = allEntities
+ .map(
+ (MapFunction) entity -> GraphCleaningFunctions
+ .applyCoarVocabularies(entity, vocs),
+ OAFENTITY_KRYO_ENC)
+ .groupByKey((MapFunction) OafEntity::getId, Encoders.STRING())
+ .reduceGroups((ReduceFunction) OafMapperUtils::mergeEntities)
+ .map(
+ (MapFunction, Tuple2>) t -> new Tuple2<>(
+ t._2().getClass().getName(), t._2()),
+ Encoders.tuple(Encoders.STRING(), OAFENTITY_KRYO_ENC));
+
+ // pivot on "_1" (classname of the entity)
+ // created columns containing only entities of the same class
+ for (Map.Entry e : ModelSupport.entityTypes.entrySet()) {
+ String entity = e.getKey().name();
+ Class entityClass = e.getValue();
+
+ groupedEntities = groupedEntities
+ .withColumn(
+ entity,
+ when(col("_1").equalTo(entityClass.getName()), col("_2")));
+ }
+
+ groupedEntities
+ .drop("_1", "_2")
+ .write()
+ .mode(SaveMode.Overwrite)
+ .option("compression", "gzip")
+ .save(checkpointPath);
+
+ ForkJoinPool parPool = new ForkJoinPool(ModelSupport.entityTypes.size());
+
+ ModelSupport.entityTypes
+ .entrySet()
+ .stream()
+ .map(e -> parPool.submit(() -> {
+ String entity = e.getKey().name();
+ Class entityClass = e.getValue();
+
+ spark
+ .read()
+ .load(checkpointPath)
+ .select(col(entity).as("value"))
+ .filter("value IS NOT NULL")
+ .as(OAFENTITY_KRYO_ENC)
+ .map((MapFunction) r -> r, (Encoder) Encoders.bean(entityClass))
+ .filter(filterInvisible ? "dataInfo.invisible != TRUE" : "TRUE")
+ .write()
+ .mode(SaveMode.Overwrite)
+ .option("compression", "gzip")
+ .json(outputPath + "/" + entity);
+ }))
+ .collect(Collectors.toList())
+ .forEach(t -> {
+ try {
+ t.get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oozie/RunSQLSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oozie/RunSQLSparkJob.java
new file mode 100644
index 000000000..027bf0735
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oozie/RunSQLSparkJob.java
@@ -0,0 +1,77 @@
+
+package eu.dnetlib.dhp.oozie;
+
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession;
+
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.commons.lang3.time.DurationFormatUtils;
+import org.apache.commons.text.StringSubstitutor;
+import org.apache.spark.SparkConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.io.Resources;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+
+public class RunSQLSparkJob {
+ private static final Logger log = LoggerFactory.getLogger(RunSQLSparkJob.class);
+
+ private final ArgumentApplicationParser parser;
+
+ public RunSQLSparkJob(ArgumentApplicationParser parser) {
+ this.parser = parser;
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ Map params = new HashMap<>();
+ for (int i = 0; i < args.length - 1; i++) {
+ if (args[i].startsWith("--")) {
+ params.put(args[i].substring(2), args[++i]);
+ }
+ }
+
+ /*
+ * String jsonConfiguration = IOUtils .toString( Objects .requireNonNull( RunSQLSparkJob.class
+ * .getResourceAsStream( "/eu/dnetlib/dhp/oozie/run_sql_parameters.json"))); final ArgumentApplicationParser
+ * parser = new ArgumentApplicationParser(jsonConfiguration); parser.parseArgument(args);
+ */
+
+ Boolean isSparkSessionManaged = Optional
+ .ofNullable(params.get("isSparkSessionManaged"))
+ .map(Boolean::valueOf)
+ .orElse(Boolean.TRUE);
+ log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+ URL url = com.google.common.io.Resources.getResource(params.get("sql"));
+ String raw_sql = Resources.toString(url, StandardCharsets.UTF_8);
+
+ String sql = StringSubstitutor.replace(raw_sql, params);
+ log.info("sql: {}", sql);
+
+ SparkConf conf = new SparkConf();
+ conf.set("hive.metastore.uris", params.get("hiveMetastoreUris"));
+
+ runWithSparkHiveSession(
+ conf,
+ isSparkSessionManaged,
+ spark -> {
+ for (String statement : sql.split(";\\s*/\\*\\s*EOS\\s*\\*/\\s*")) {
+ log.info("executing: {}", statement);
+ long startTime = System.currentTimeMillis();
+ spark.sql(statement).show();
+ log
+ .info(
+ "executed in {}",
+ DurationFormatUtils.formatDuration(System.currentTimeMillis() - startTime, "HH:mm:ss.S"));
+ }
+ });
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/parser/utility/VtdUtilityParser.java b/dhp-common/src/main/java/eu/dnetlib/dhp/parser/utility/VtdUtilityParser.java
index 9ac0a0bf7..fd4c0191a 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/parser/utility/VtdUtilityParser.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/parser/utility/VtdUtilityParser.java
@@ -12,6 +12,9 @@ import com.ximpleware.VTDNav;
/** Created by sandro on 9/29/16. */
public class VtdUtilityParser {
+ private VtdUtilityParser() {
+ }
+
public static List getTextValuesWithAttributes(
final AutoPilot ap, final VTDNav vn, final String xpath, final List attributes)
throws VtdException {
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/OafMapperUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/OafMapperUtils.java
deleted file mode 100644
index 4a66f91dc..000000000
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/OafMapperUtils.java
+++ /dev/null
@@ -1,297 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.Function;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang3.StringUtils;
-
-import eu.dnetlib.dhp.schema.common.ModelSupport;
-import eu.dnetlib.dhp.utils.DHPUtils;
-
-public class OafMapperUtils {
-
- public static Oaf merge(final Oaf o1, final Oaf o2) {
- if (ModelSupport.isSubClass(o1, OafEntity.class)) {
- if (ModelSupport.isSubClass(o1, Result.class)) {
-
- return mergeResults((Result) o1, (Result) o2);
- } else if (ModelSupport.isSubClass(o1, Datasource.class)) {
- ((Datasource) o1).mergeFrom((Datasource) o2);
- } else if (ModelSupport.isSubClass(o1, Organization.class)) {
- ((Organization) o1).mergeFrom((Organization) o2);
- } else if (ModelSupport.isSubClass(o1, Project.class)) {
- ((Project) o1).mergeFrom((Project) o2);
- } else {
- throw new RuntimeException("invalid OafEntity subtype:" + o1.getClass().getCanonicalName());
- }
- } else if (ModelSupport.isSubClass(o1, Relation.class)) {
- ((Relation) o1).mergeFrom((Relation) o2);
- } else {
- throw new RuntimeException("invalid Oaf type:" + o1.getClass().getCanonicalName());
- }
- return o1;
- }
-
- public static Result mergeResults(Result r1, Result r2) {
- if (new ResultTypeComparator().compare(r1, r2) < 0) {
- r1.mergeFrom(r2);
- return r1;
- } else {
- r2.mergeFrom(r1);
- return r2;
- }
- }
-
- public static KeyValue keyValue(final String k, final String v) {
- final KeyValue kv = new KeyValue();
- kv.setKey(k);
- kv.setValue(v);
- return kv;
- }
-
- public static List listKeyValues(final String... s) {
- if (s.length % 2 > 0) {
- throw new RuntimeException("Invalid number of parameters (k,v,k,v,....)");
- }
-
- final List list = new ArrayList<>();
- for (int i = 0; i < s.length; i += 2) {
- list.add(keyValue(s[i], s[i + 1]));
- }
- return list;
- }
-
- public static Field field(final T value, final DataInfo info) {
- if (value == null || StringUtils.isBlank(value.toString())) {
- return null;
- }
-
- final Field field = new Field<>();
- field.setValue(value);
- field.setDataInfo(info);
- return field;
- }
-
- public static List> listFields(final DataInfo info, final String... values) {
- return Arrays
- .stream(values)
- .map(v -> field(v, info))
- .filter(Objects::nonNull)
- .filter(distinctByKey(f -> f.getValue()))
- .collect(Collectors.toList());
- }
-
- public static List> listFields(final DataInfo info, final List values) {
- return values
- .stream()
- .map(v -> field(v, info))
- .filter(Objects::nonNull)
- .filter(distinctByKey(f -> f.getValue()))
- .collect(Collectors.toList());
- }
-
- public static Qualifier unknown(final String schemeid, final String schemename) {
- return qualifier("UNKNOWN", "Unknown", schemeid, schemename);
- }
-
- public static Qualifier qualifier(
- final String classid,
- final String classname,
- final String schemeid,
- final String schemename) {
- final Qualifier q = new Qualifier();
- q.setClassid(classid);
- q.setClassname(classname);
- q.setSchemeid(schemeid);
- q.setSchemename(schemename);
- return q;
- }
-
- public static StructuredProperty structuredProperty(
- final String value,
- final String classid,
- final String classname,
- final String schemeid,
- final String schemename,
- final DataInfo dataInfo) {
-
- return structuredProperty(value, qualifier(classid, classname, schemeid, schemename), dataInfo);
- }
-
- public static StructuredProperty structuredProperty(
- final String value,
- final Qualifier qualifier,
- final DataInfo dataInfo) {
- if (value == null) {
- return null;
- }
- final StructuredProperty sp = new StructuredProperty();
- sp.setValue(value);
- sp.setQualifier(qualifier);
- sp.setDataInfo(dataInfo);
- return sp;
- }
-
- public static ExtraInfo extraInfo(
- final String name,
- final String value,
- final String typology,
- final String provenance,
- final String trust) {
- final ExtraInfo info = new ExtraInfo();
- info.setName(name);
- info.setValue(value);
- info.setTypology(typology);
- info.setProvenance(provenance);
- info.setTrust(trust);
- return info;
- }
-
- public static OAIProvenance oaiIProvenance(
- final String identifier,
- final String baseURL,
- final String metadataNamespace,
- final Boolean altered,
- final String datestamp,
- final String harvestDate) {
-
- final OriginDescription desc = new OriginDescription();
- desc.setIdentifier(identifier);
- desc.setBaseURL(baseURL);
- desc.setMetadataNamespace(metadataNamespace);
- desc.setAltered(altered);
- desc.setDatestamp(datestamp);
- desc.setHarvestDate(harvestDate);
-
- final OAIProvenance p = new OAIProvenance();
- p.setOriginDescription(desc);
-
- return p;
- }
-
- public static Journal journal(
- final String name,
- final String issnPrinted,
- final String issnOnline,
- final String issnLinking,
- final DataInfo dataInfo) {
- return journal(
- name,
- issnPrinted,
- issnOnline,
- issnLinking,
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- dataInfo);
- }
-
- public static Journal journal(
- final String name,
- final String issnPrinted,
- final String issnOnline,
- final String issnLinking,
- final String ep,
- final String iss,
- final String sp,
- final String vol,
- final String edition,
- final String conferenceplace,
- final String conferencedate,
- final DataInfo dataInfo) {
-
- if (StringUtils.isNotBlank(name)
- || StringUtils.isNotBlank(issnPrinted)
- || StringUtils.isNotBlank(issnOnline)
- || StringUtils.isNotBlank(issnLinking)) {
- final Journal j = new Journal();
- j.setName(name);
- j.setIssnPrinted(issnPrinted);
- j.setIssnOnline(issnOnline);
- j.setIssnLinking(issnLinking);
- j.setEp(ep);
- j.setIss(iss);
- j.setSp(sp);
- j.setVol(vol);
- j.setEdition(edition);
- j.setConferenceplace(conferenceplace);
- j.setConferencedate(conferencedate);
- j.setDataInfo(dataInfo);
- return j;
- } else {
- return null;
- }
- }
-
- public static DataInfo dataInfo(
- final Boolean deletedbyinference,
- final String inferenceprovenance,
- final Boolean inferred,
- final Boolean invisible,
- final Qualifier provenanceaction,
- final String trust) {
- final DataInfo d = new DataInfo();
- d.setDeletedbyinference(deletedbyinference);
- d.setInferenceprovenance(inferenceprovenance);
- d.setInferred(inferred);
- d.setInvisible(invisible);
- d.setProvenanceaction(provenanceaction);
- d.setTrust(trust);
- return d;
- }
-
- public static String createOpenaireId(
- final int prefix,
- final String originalId,
- final boolean to_md5) {
- if (StringUtils.isBlank(originalId)) {
- return null;
- } else if (to_md5) {
- final String nsPrefix = StringUtils.substringBefore(originalId, "::");
- final String rest = StringUtils.substringAfter(originalId, "::");
- return String.format("%s|%s::%s", prefix, nsPrefix, DHPUtils.md5(rest));
- } else {
- return String.format("%s|%s", prefix, originalId);
- }
- }
-
- public static String createOpenaireId(
- final String type,
- final String originalId,
- final boolean to_md5) {
- switch (type) {
- case "datasource":
- return createOpenaireId(10, originalId, to_md5);
- case "organization":
- return createOpenaireId(20, originalId, to_md5);
- case "person":
- return createOpenaireId(30, originalId, to_md5);
- case "project":
- return createOpenaireId(40, originalId, to_md5);
- default:
- return createOpenaireId(50, originalId, to_md5);
- }
- }
-
- public static String asString(final Object o) {
- return o == null ? "" : o.toString();
- }
-
- public static Predicate distinctByKey(
- final Function keyExtractor) {
- final Map seen = new ConcurrentHashMap<>();
- return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
- }
-}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/ResultTypeComparator.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/ResultTypeComparator.java
deleted file mode 100644
index 6c11d1a85..000000000
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/ResultTypeComparator.java
+++ /dev/null
@@ -1,49 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.util.Comparator;
-
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-
-public class ResultTypeComparator implements Comparator {
-
- @Override
- public int compare(Result left, Result right) {
-
- if (left == null && right == null)
- return 0;
- if (left == null)
- return 1;
- if (right == null)
- return -1;
-
- String lClass = left.getResulttype().getClassid();
- String rClass = right.getResulttype().getClassid();
-
- if (lClass.equals(rClass))
- return 0;
-
- if (lClass.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID))
- return -1;
- if (rClass.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID))
- return 1;
-
- if (lClass.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID))
- return -1;
- if (rClass.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID))
- return 1;
-
- if (lClass.equals(ModelConstants.SOFTWARE_RESULTTYPE_CLASSID))
- return -1;
- if (rClass.equals(ModelConstants.SOFTWARE_RESULTTYPE_CLASSID))
- return 1;
-
- if (lClass.equals(ModelConstants.ORP_RESULTTYPE_CLASSID))
- return -1;
- if (rClass.equals(ModelConstants.ORP_RESULTTYPE_CLASSID))
- return 1;
-
- // Else (but unlikely), lexicographical ordering will do.
- return lClass.compareTo(rClass);
- }
-}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/DoiCleaningRule.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/DoiCleaningRule.java
new file mode 100644
index 000000000..1a7482685
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/DoiCleaningRule.java
@@ -0,0 +1,14 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+public class DoiCleaningRule {
+
+ public static String clean(final String doi) {
+ return doi
+ .toLowerCase()
+ .replaceAll("\\s", "")
+ .replaceAll("^doi:", "")
+ .replaceFirst(CleaningFunctions.DOI_PREFIX_REGEX, CleaningFunctions.DOI_PREFIX);
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/FundRefCleaningRule.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/FundRefCleaningRule.java
new file mode 100644
index 000000000..a267b8b88
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/FundRefCleaningRule.java
@@ -0,0 +1,25 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class FundRefCleaningRule {
+
+ public static final Pattern PATTERN = Pattern.compile("\\d+");
+
+ public static String clean(final String fundRefId) {
+
+ String s = fundRefId
+ .toLowerCase()
+ .replaceAll("\\s", "");
+
+ Matcher m = PATTERN.matcher(s);
+ if (m.find()) {
+ return m.group();
+ } else {
+ return "";
+ }
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java
new file mode 100644
index 000000000..f01f90fe4
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java
@@ -0,0 +1,1002 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
+import static eu.dnetlib.dhp.schema.common.ModelConstants.OPENAIRE_META_RESOURCE_TYPE;
+import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.getProvenance;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.github.sisyphsu.dateparser.DateParserUtils;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
+import eu.dnetlib.dhp.common.vocabulary.VocabularyTerm;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.common.ModelSupport;
+import eu.dnetlib.dhp.schema.oaf.*;
+import me.xuender.unidecode.Unidecode;
+
+public class GraphCleaningFunctions extends CleaningFunctions {
+
+ public static final String DNET_PUBLISHERS = "dnet:publishers";
+
+ public static final String DNET_LICENSES = "dnet:licenses";
+
+ public static final String ORCID_CLEANING_REGEX = ".*([0-9]{4}).*[-–—−=].*([0-9]{4}).*[-–—−=].*([0-9]{4}).*[-–—−=].*([0-9x]{4})";
+ public static final int ORCID_LEN = 19;
+ public static final String CLEANING_REGEX = "(?:\\n|\\r|\\t)";
+ public static final String INVALID_AUTHOR_REGEX = ".*deactivated.*";
+
+ public static final String TITLE_TEST = "test";
+ public static final String TITLE_FILTER_REGEX = String.format("(%s)|\\W|\\d", TITLE_TEST);
+
+ public static final int TITLE_FILTER_RESIDUAL_LENGTH = 5;
+ private static final String NAME_CLEANING_REGEX = "[\\r\\n\\t\\s]+";
+
+ private static final Set INVALID_AUTHOR_NAMES = new HashSet<>();
+
+ private static final Set INVALID_URLS = new HashSet<>();
+
+ private static final Set INVALID_URL_HOSTS = new HashSet<>();
+
+ private static final HashSet PEER_REVIEWED_TYPES = new HashSet<>();
+
+ static {
+ PEER_REVIEWED_TYPES.add("Article");
+ PEER_REVIEWED_TYPES.add("Part of book or chapter of book");
+ PEER_REVIEWED_TYPES.add("Book");
+ PEER_REVIEWED_TYPES.add("Doctoral thesis");
+ PEER_REVIEWED_TYPES.add("Master thesis");
+ PEER_REVIEWED_TYPES.add("Data Paper");
+ PEER_REVIEWED_TYPES.add("Thesis");
+ PEER_REVIEWED_TYPES.add("Bachelor thesis");
+ PEER_REVIEWED_TYPES.add("Conference object");
+
+ INVALID_AUTHOR_NAMES.add("(:null)");
+ INVALID_AUTHOR_NAMES.add("(:unap)");
+ INVALID_AUTHOR_NAMES.add("(:tba)");
+ INVALID_AUTHOR_NAMES.add("(:unas)");
+ INVALID_AUTHOR_NAMES.add("(:unav)");
+ INVALID_AUTHOR_NAMES.add("(:unkn)");
+ INVALID_AUTHOR_NAMES.add("(:unkn) unknown");
+ INVALID_AUTHOR_NAMES.add(":none");
+ INVALID_AUTHOR_NAMES.add(":null");
+ INVALID_AUTHOR_NAMES.add(":unas");
+ INVALID_AUTHOR_NAMES.add(":unav");
+ INVALID_AUTHOR_NAMES.add(":unkn");
+ INVALID_AUTHOR_NAMES.add("[autor desconocido]");
+ INVALID_AUTHOR_NAMES.add("[s. n.]");
+ INVALID_AUTHOR_NAMES.add("[s.n]");
+ INVALID_AUTHOR_NAMES.add("[unknown]");
+ INVALID_AUTHOR_NAMES.add("anonymous");
+ INVALID_AUTHOR_NAMES.add("n.n.");
+ INVALID_AUTHOR_NAMES.add("nn");
+ INVALID_AUTHOR_NAMES.add("no name supplied");
+ INVALID_AUTHOR_NAMES.add("none");
+ INVALID_AUTHOR_NAMES.add("none available");
+ INVALID_AUTHOR_NAMES.add("not available not available");
+ INVALID_AUTHOR_NAMES.add("null &na;");
+ INVALID_AUTHOR_NAMES.add("null anonymous");
+ INVALID_AUTHOR_NAMES.add("unbekannt");
+ INVALID_AUTHOR_NAMES.add("unknown");
+
+ INVALID_URL_HOSTS.add("creativecommons.org");
+ INVALID_URL_HOSTS.add("www.academia.edu");
+ INVALID_URL_HOSTS.add("academia.edu");
+ INVALID_URL_HOSTS.add("researchgate.net");
+ INVALID_URL_HOSTS.add("www.researchgate.net");
+
+ INVALID_URLS.add("http://repo.scoap3.org/api");
+ INVALID_URLS.add("http://ora.ox.ac.uk/objects/uuid:");
+ INVALID_URLS.add("http://ntur.lib.ntu.edu.tw/news/agent_contract.pdf");
+ INVALID_URLS.add("https://media.springer.com/full/springer-instructions-for-authors-assets/pdf/SN_BPF_EN.pdf");
+ INVALID_URLS.add("http://www.tobaccoinduceddiseases.org/dl/61aad426c96519bea4040a374c6a6110/");
+ INVALID_URLS.add("https://www.bilboard.nl/verenigingsbladen/bestuurskundige-berichten");
+ }
+
+ public static T cleanContext(T value, String contextId, String verifyParam) {
+ if (ModelSupport.isSubClass(value, Result.class)) {
+ final Result res = (Result) value;
+ if (shouldCleanContext(res, verifyParam)) {
+ res
+ .setContext(
+ res
+ .getContext()
+ .stream()
+ .filter(c -> !StringUtils.startsWith(c.getId().toLowerCase(), contextId))
+ .collect(Collectors.toList()));
+ }
+ return (T) res;
+ } else {
+ return value;
+ }
+ }
+
+ private static boolean shouldCleanContext(Result res, String verifyParam) {
+ boolean titleMatch = res
+ .getTitle()
+ .stream()
+ .filter(
+ t -> t
+ .getQualifier()
+ .getClassid()
+ .equalsIgnoreCase(ModelConstants.MAIN_TITLE_QUALIFIER.getClassid()))
+ .anyMatch(t -> t.getValue().toLowerCase().startsWith(verifyParam.toLowerCase()));
+
+ return titleMatch && Objects.nonNull(res.getContext());
+ }
+
+ public static T cleanCountry(T value, String[] verifyParam, Set hostedBy,
+ String collectedfrom, String country) {
+ if (ModelSupport.isSubClass(value, Result.class)) {
+ final Result res = (Result) value;
+ if (res.getInstance().stream().anyMatch(i -> hostedBy.contains(i.getHostedby().getKey())) ||
+ !res.getCollectedfrom().stream().anyMatch(cf -> cf.getValue().equals(collectedfrom))) {
+ return (T) res;
+ }
+
+ List ids = getPidsAndAltIds(res).collect(Collectors.toList());
+ if (ids
+ .stream()
+ .anyMatch(
+ p -> p
+ .getQualifier()
+ .getClassid()
+ .equals(PidType.doi.toString()) && pidInParam(p.getValue(), verifyParam))) {
+ res
+ .setCountry(
+ res
+ .getCountry()
+ .stream()
+ .filter(
+ c -> toTakeCountry(c, country))
+ .collect(Collectors.toList()));
+ }
+
+ return (T) res;
+ } else {
+ return value;
+ }
+ }
+
+ private static Stream getPidsAndAltIds(T r) {
+ final Stream resultPids = Optional
+ .ofNullable(r.getPid())
+ .map(Collection::stream)
+ .orElse(Stream.empty());
+
+ final Stream instancePids = Optional
+ .ofNullable(r.getInstance())
+ .map(
+ instance -> instance
+ .stream()
+ .flatMap(
+ i -> Optional
+ .ofNullable(i.getPid())
+ .map(Collection::stream)
+ .orElse(Stream.empty())))
+ .orElse(Stream.empty());
+
+ final Stream instanceAltIds = Optional
+ .ofNullable(r.getInstance())
+ .map(
+ instance -> instance
+ .stream()
+ .flatMap(
+ i -> Optional
+ .ofNullable(i.getAlternateIdentifier())
+ .map(Collection::stream)
+ .orElse(Stream.empty())))
+ .orElse(Stream.empty());
+
+ return Stream
+ .concat(
+ Stream.concat(resultPids, instancePids),
+ instanceAltIds);
+ }
+
+ private static boolean pidInParam(String value, String[] verifyParam) {
+ for (String s : verifyParam)
+ if (value.startsWith(s))
+ return true;
+ return false;
+ }
+
+ private static boolean toTakeCountry(Country c, String country) {
+ // If dataInfo is not set, or dataInfo.inferenceprovenance is not set or not present then it cannot be
+ // inserted via propagation
+ if (!Optional.ofNullable(c.getDataInfo()).isPresent())
+ return true;
+ if (!Optional.ofNullable(c.getDataInfo().getInferenceprovenance()).isPresent())
+ return true;
+ return !(c
+ .getClassid()
+ .equalsIgnoreCase(country) &&
+ c.getDataInfo().getInferenceprovenance().equals("propagation"));
+ }
+
+ public static T fixVocabularyNames(T value) {
+ if (value instanceof OafEntity) {
+
+ OafEntity e = (OafEntity) value;
+
+ Optional
+ .ofNullable(e.getPid())
+ .ifPresent(pid -> pid.forEach(p -> fixVocabName(p.getQualifier(), ModelConstants.DNET_PID_TYPES)));
+
+ if (value instanceof Result) {
+ Result r = (Result) value;
+
+ fixVocabName(r.getLanguage(), ModelConstants.DNET_LANGUAGES);
+ fixVocabName(r.getResourcetype(), ModelConstants.DNET_DATA_CITE_RESOURCE);
+ fixVocabName(r.getBestaccessright(), ModelConstants.DNET_ACCESS_MODES);
+
+ if (Objects.nonNull(r.getSubject())) {
+ r.getSubject().forEach(s -> fixVocabName(s.getQualifier(), ModelConstants.DNET_SUBJECT_TYPOLOGIES));
+ }
+ if (Objects.nonNull(r.getInstance())) {
+ for (Instance i : r.getInstance()) {
+ fixVocabName(i.getAccessright(), ModelConstants.DNET_ACCESS_MODES);
+ fixVocabName(i.getRefereed(), ModelConstants.DNET_REVIEW_LEVELS);
+ Optional
+ .ofNullable(i.getPid())
+ .ifPresent(
+ pid -> pid.forEach(p -> fixVocabName(p.getQualifier(), ModelConstants.DNET_PID_TYPES)));
+
+ }
+ }
+ if (Objects.nonNull(r.getAuthor())) {
+ r.getAuthor().stream().filter(Objects::nonNull).forEach(a -> {
+ if (Objects.nonNull(a.getPid())) {
+ a.getPid().stream().filter(Objects::nonNull).forEach(p -> {
+ fixVocabName(p.getQualifier(), ModelConstants.DNET_PID_TYPES);
+ });
+ }
+ });
+ }
+ if (value instanceof Publication) {
+
+ } else if (value instanceof Dataset) {
+
+ } else if (value instanceof OtherResearchProduct) {
+
+ } else if (value instanceof Software) {
+
+ }
+ } else if (value instanceof Datasource) {
+ // nothing to clean here
+ } else if (value instanceof Project) {
+ // nothing to clean here
+ } else if (value instanceof Organization) {
+ Organization o = (Organization) value;
+ if (Objects.nonNull(o.getCountry())) {
+ fixVocabName(o.getCountry(), ModelConstants.DNET_COUNTRY_TYPE);
+ }
+
+ }
+ } else if (value instanceof Relation) {
+ // nothing to clean here
+ }
+
+ return value;
+ }
+
+ public static boolean filter(T value) {
+ if (!(value instanceof Relation) && (Boolean.TRUE
+ .equals(
+ Optional
+ .ofNullable(value)
+ .map(
+ o -> Optional
+ .ofNullable(o.getDataInfo())
+ .map(
+ d -> Optional
+ .ofNullable(d.getInvisible())
+ .orElse(true))
+ .orElse(false))
+ .orElse(true)))) {
+ return true;
+ }
+
+ if (value instanceof Datasource) {
+ final Datasource d = (Datasource) value;
+ return Objects.nonNull(d.getOfficialname()) && StringUtils.isNotBlank(d.getOfficialname().getValue());
+ } else if (value instanceof Project) {
+ final Project p = (Project) value;
+ return Objects.nonNull(p.getCode()) && StringUtils.isNotBlank(p.getCode().getValue());
+ } else if (value instanceof Organization) {
+ // nothing to evaluate here
+ } else if (value instanceof Relation) {
+ // nothing to clean here
+ } else if (value instanceof Result) {
+
+ Result r = (Result) value;
+
+ if (Objects.isNull(r.getTitle()) || r.getTitle().isEmpty()) {
+ return false;
+ }
+
+ if (value instanceof Publication) {
+
+ } else if (value instanceof Dataset) {
+
+ } else if (value instanceof OtherResearchProduct) {
+
+ } else if (value instanceof Software) {
+
+ }
+ }
+ return true;
+ }
+
+ public static T cleanup(T value, VocabularyGroup vocs) {
+
+ if (Objects.isNull(value.getDataInfo())) {
+ final DataInfo d = new DataInfo();
+ d.setDeletedbyinference(false);
+ value.setDataInfo(d);
+ }
+
+ if (value instanceof OafEntity) {
+
+ OafEntity e = (OafEntity) value;
+ if (Objects.nonNull(e.getPid())) {
+ e.setPid(processPidCleaning(e.getPid()));
+ }
+
+ if (value instanceof Datasource) {
+ // nothing to clean here
+ } else if (value instanceof Project) {
+ // nothing to clean here
+ } else if (value instanceof Organization) {
+ Organization o = (Organization) value;
+ if (Objects.isNull(o.getCountry()) || StringUtils.isBlank(o.getCountry().getClassid())) {
+ o.setCountry(ModelConstants.UNKNOWN_COUNTRY);
+ }
+ } else if (value instanceof Result) {
+ Result r = (Result) value;
+
+ if (Objects.isNull(r.getContext())) {
+ r.setContext(new ArrayList<>());
+ }
+
+ if (Objects.nonNull(r.getFulltext())
+ && (ModelConstants.SOFTWARE_RESULTTYPE_CLASSID.equals(r.getResulttype().getClassid()) ||
+ ModelConstants.DATASET_RESULTTYPE_CLASSID.equals(r.getResulttype().getClassid()))) {
+ r.setFulltext(null);
+
+ }
+
+ if (Objects.nonNull(r.getDateofacceptance())) {
+ Optional date = cleanDateField(r.getDateofacceptance());
+ if (date.isPresent()) {
+ r.getDateofacceptance().setValue(date.get());
+ } else {
+ r.setDateofacceptance(null);
+ }
+ }
+ if (Objects.nonNull(r.getRelevantdate())) {
+ r
+ .setRelevantdate(
+ r
+ .getRelevantdate()
+ .stream()
+ .filter(Objects::nonNull)
+ .filter(sp -> Objects.nonNull(sp.getQualifier()))
+ .filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
+ .map(sp -> {
+ sp.setValue(GraphCleaningFunctions.cleanDate(sp.getValue()));
+ return sp;
+ })
+ .filter(sp -> StringUtils.isNotBlank(sp.getValue()))
+ .collect(Collectors.toList()));
+ }
+ if (Objects.nonNull(r.getPublisher())) {
+ if (StringUtils.isBlank(r.getPublisher().getValue())) {
+ r.setPublisher(null);
+ } else {
+ r
+ .getPublisher()
+ .setValue(
+ r
+ .getPublisher()
+ .getValue()
+ .replaceAll(NAME_CLEANING_REGEX, " "));
+
+ if (vocs.vocabularyExists(DNET_PUBLISHERS)) {
+ vocs
+ .find(DNET_PUBLISHERS)
+ .map(voc -> voc.getTermBySynonym(r.getPublisher().getValue()))
+ .map(VocabularyTerm::getName)
+ .ifPresent(publisher -> r.getPublisher().setValue(publisher));
+ }
+ }
+ }
+ if (Objects.isNull(r.getLanguage()) || StringUtils.isBlank(r.getLanguage().getClassid())) {
+ r
+ .setLanguage(
+ qualifier("und", "Undetermined", ModelConstants.DNET_LANGUAGES));
+ }
+ if (Objects.nonNull(r.getSubject())) {
+ List subjects = Lists
+ .newArrayList(
+ r
+ .getSubject()
+ .stream()
+ .filter(Objects::nonNull)
+ .filter(sp -> StringUtils.isNotBlank(sp.getValue()))
+ .filter(sp -> Objects.nonNull(sp.getQualifier()))
+ .filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
+ .map(s -> {
+ if ("dnet:result_subject".equals(s.getQualifier().getClassid())) {
+ s.getQualifier().setClassid(ModelConstants.DNET_SUBJECT_TYPOLOGIES);
+ s.getQualifier().setClassname(ModelConstants.DNET_SUBJECT_TYPOLOGIES);
+ }
+ return s;
+ })
+ .map(GraphCleaningFunctions::cleanValue)
+ .collect(
+ Collectors
+ .toMap(
+ s -> Optional
+ .ofNullable(s.getQualifier())
+ .map(q -> q.getClassid() + s.getValue())
+ .orElse(s.getValue()),
+ Function.identity(),
+ (s1, s2) -> Collections
+ .min(Lists.newArrayList(s1, s2), new SubjectProvenanceComparator())))
+ .values());
+ r.setSubject(subjects);
+ }
+ if (Objects.nonNull(r.getTitle())) {
+ r
+ .setTitle(
+ r
+ .getTitle()
+ .stream()
+ .filter(Objects::nonNull)
+ .filter(sp -> StringUtils.isNotBlank(sp.getValue()))
+ .filter(
+ sp -> {
+ final String title = sp
+ .getValue()
+ .toLowerCase();
+ final String decoded = Unidecode.decode(title);
+
+ if (StringUtils.contains(decoded, TITLE_TEST)) {
+ return decoded
+ .replaceAll(TITLE_FILTER_REGEX, "")
+ .length() > TITLE_FILTER_RESIDUAL_LENGTH;
+ }
+ return !decoded
+ .replaceAll("\\W|\\d", "")
+ .isEmpty();
+ })
+ .map(GraphCleaningFunctions::cleanValue)
+ .collect(Collectors.toList()));
+ }
+ if (Objects.nonNull(r.getFormat())) {
+ r
+ .setFormat(
+ r
+ .getFormat()
+ .stream()
+ .map(GraphCleaningFunctions::cleanValue)
+ .collect(Collectors.toList()));
+ }
+ if (Objects.nonNull(r.getDescription())) {
+ r
+ .setDescription(
+ r
+ .getDescription()
+ .stream()
+ .filter(Objects::nonNull)
+ .filter(sp -> StringUtils.isNotBlank(sp.getValue()))
+ .map(GraphCleaningFunctions::cleanValue)
+ .collect(Collectors.toList()));
+ }
+ if (Objects.isNull(r.getResourcetype()) || StringUtils.isBlank(r.getResourcetype().getClassid())) {
+ r
+ .setResourcetype(
+ qualifier(ModelConstants.UNKNOWN, "Unknown", ModelConstants.DNET_DATA_CITE_RESOURCE));
+ }
+ if (Objects.nonNull(r.getInstance())) {
+
+ for (Instance i : r.getInstance()) {
+ if (!vocs
+ .termExists(ModelConstants.DNET_PUBLICATION_RESOURCE, i.getInstancetype().getClassid())) {
+ if (r instanceof Publication) {
+ i
+ .setInstancetype(
+ OafMapperUtils
+ .qualifier(
+ "0038", "Other literature type",
+ ModelConstants.DNET_PUBLICATION_RESOURCE,
+ ModelConstants.DNET_PUBLICATION_RESOURCE));
+ } else if (r instanceof Dataset) {
+ i
+ .setInstancetype(
+ OafMapperUtils
+ .qualifier(
+ "0039", "Other dataset type", ModelConstants.DNET_PUBLICATION_RESOURCE,
+ ModelConstants.DNET_PUBLICATION_RESOURCE));
+ } else if (r instanceof Software) {
+ i
+ .setInstancetype(
+ OafMapperUtils
+ .qualifier(
+ "0040", "Other software type", ModelConstants.DNET_PUBLICATION_RESOURCE,
+ ModelConstants.DNET_PUBLICATION_RESOURCE));
+ } else if (r instanceof OtherResearchProduct) {
+ i
+ .setInstancetype(
+ OafMapperUtils
+ .qualifier(
+ "0020", "Other ORP type", ModelConstants.DNET_PUBLICATION_RESOURCE,
+ ModelConstants.DNET_PUBLICATION_RESOURCE));
+ }
+ }
+
+ if (Objects.nonNull(i.getPid())) {
+ i.setPid(processPidCleaning(i.getPid()));
+ }
+ if (Objects.nonNull(i.getAlternateIdentifier())) {
+ i.setAlternateIdentifier(processPidCleaning(i.getAlternateIdentifier()));
+ }
+ Optional
+ .ofNullable(i.getPid())
+ .ifPresent(pid -> {
+ final Set pids = Sets.newHashSet(pid);
+ Optional
+ .ofNullable(i.getAlternateIdentifier())
+ .ifPresent(altId -> {
+ final Set altIds = Sets.newHashSet(altId);
+ i.setAlternateIdentifier(Lists.newArrayList(Sets.difference(altIds, pids)));
+ });
+ });
+
+ if (Objects.isNull(i.getAccessright())
+ || StringUtils.isBlank(i.getAccessright().getClassid())) {
+ i
+ .setAccessright(
+ accessRight(
+ ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE,
+ ModelConstants.DNET_ACCESS_MODES));
+ }
+ if (Objects.isNull(i.getHostedby()) || StringUtils.isBlank(i.getHostedby().getKey())) {
+ i.setHostedby(ModelConstants.UNKNOWN_REPOSITORY);
+ }
+ if (Objects.isNull(i.getRefereed()) || StringUtils.isBlank(i.getRefereed().getClassid())) {
+ i.setRefereed(qualifier("0000", "Unknown", ModelConstants.DNET_REVIEW_LEVELS));
+ }
+
+ if (Objects.nonNull(i.getLicense()) && Objects.nonNull(i.getLicense().getValue())) {
+ vocs
+ .find(DNET_LICENSES)
+ .map(voc -> voc.getTermBySynonym(i.getLicense().getValue()))
+ .map(VocabularyTerm::getId)
+ .ifPresent(license -> i.getLicense().setValue(license));
+ }
+
+ // from the script from Dimitris
+ if ("0000".equals(i.getRefereed().getClassid())) {
+ final boolean isFromCrossref = Optional
+ .ofNullable(i.getCollectedfrom())
+ .map(KeyValue::getKey)
+ .map(id -> id.equals(ModelConstants.CROSSREF_ID))
+ .orElse(false);
+ final boolean hasDoi = Optional
+ .ofNullable(i.getPid())
+ .map(
+ pid -> pid
+ .stream()
+ .anyMatch(
+ p -> PidType.doi.toString().equals(p.getQualifier().getClassid())))
+ .orElse(false);
+ final boolean isPeerReviewedType = PEER_REVIEWED_TYPES
+ .contains(i.getInstancetype().getClassname());
+ final boolean noOtherLitType = r
+ .getInstance()
+ .stream()
+ .noneMatch(ii -> "Other literature type".equals(ii.getInstancetype().getClassname()));
+ if (isFromCrossref && hasDoi && isPeerReviewedType && noOtherLitType) {
+ i.setRefereed(qualifier("0001", "peerReviewed", ModelConstants.DNET_REVIEW_LEVELS));
+ } else {
+ i.setRefereed(qualifier("0002", "nonPeerReviewed", ModelConstants.DNET_REVIEW_LEVELS));
+ }
+ }
+
+ if (Objects.nonNull(i.getDateofacceptance())) {
+ Optional date = cleanDateField(i.getDateofacceptance());
+ if (date.isPresent()) {
+ i.getDateofacceptance().setValue(date.get());
+ } else {
+ i.setDateofacceptance(null);
+ }
+ }
+ if (StringUtils.isNotBlank(i.getFulltext()) &&
+ (ModelConstants.SOFTWARE_RESULTTYPE_CLASSID.equals(r.getResulttype().getClassid()) ||
+ ModelConstants.DATASET_RESULTTYPE_CLASSID.equals(r.getResulttype().getClassid()))) {
+ i.setFulltext(null);
+ }
+ if (Objects.nonNull(i.getUrl())) {
+ i
+ .setUrl(
+ i
+ .getUrl()
+ .stream()
+ .filter(GraphCleaningFunctions::urlFilter)
+ .collect(Collectors.toList()));
+ }
+ }
+ }
+ if (Objects.isNull(r.getBestaccessright())
+ || StringUtils.isBlank(r.getBestaccessright().getClassid())) {
+ Qualifier bestaccessrights = OafMapperUtils.createBestAccessRights(r.getInstance());
+ if (Objects.isNull(bestaccessrights)) {
+ r
+ .setBestaccessright(
+ qualifier(
+ ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE,
+ ModelConstants.DNET_ACCESS_MODES));
+ } else {
+ r.setBestaccessright(bestaccessrights);
+ }
+ }
+ if (Objects.nonNull(r.getAuthor())) {
+ r
+ .setAuthor(
+ r
+ .getAuthor()
+ .stream()
+ .filter(Objects::nonNull)
+ .filter(GraphCleaningFunctions::isValidAuthorName)
+ .map(GraphCleaningFunctions::cleanupAuthor)
+ .collect(Collectors.toList()));
+
+ boolean nullRank = r
+ .getAuthor()
+ .stream()
+ .anyMatch(a -> Objects.isNull(a.getRank()));
+ if (nullRank) {
+ int i = 1;
+ for (Author author : r.getAuthor()) {
+ author.setRank(i++);
+ }
+ }
+
+ for (Author a : r.getAuthor()) {
+ if (Objects.isNull(a.getPid())) {
+ a.setPid(Lists.newArrayList());
+ } else {
+ a
+ .setPid(
+ a
+ .getPid()
+ .stream()
+ .filter(Objects::nonNull)
+ .filter(p -> Objects.nonNull(p.getQualifier()))
+ .filter(p -> StringUtils.isNotBlank(p.getValue()))
+ .filter(
+ p -> StringUtils
+ .contains(StringUtils.lowerCase(p.getQualifier().getClassid()), ORCID))
+ .map(p -> {
+ // hack to distinguish orcid from orcid_pending
+ String pidProvenance = getProvenance(p.getDataInfo());
+ if (p
+ .getQualifier()
+ .getClassid()
+ .toLowerCase()
+ .contains(ModelConstants.ORCID)) {
+ if (pidProvenance
+ .equals(ModelConstants.SYSIMPORT_CROSSWALK_ENTITYREGISTRY) ||
+ pidProvenance.equals("ORCID_ENRICHMENT")) {
+ p.getQualifier().setClassid(ModelConstants.ORCID);
+ } else {
+ p.getQualifier().setClassid(ModelConstants.ORCID_PENDING);
+ }
+ final String orcid = p
+ .getValue()
+ .trim()
+ .toLowerCase()
+ .replaceAll(ORCID_CLEANING_REGEX, "$1-$2-$3-$4");
+ if (orcid.length() == ORCID_LEN) {
+ p.setValue(orcid);
+ } else {
+ p.setValue("");
+ }
+ }
+ return p;
+ })
+ .filter(p -> StringUtils.isNotBlank(p.getValue()))
+ .collect(
+ Collectors
+ .toMap(
+ p -> p.getQualifier().getClassid() + p.getValue(),
+ Function.identity(),
+ (p1, p2) -> p1,
+ LinkedHashMap::new))
+ .values()
+ .stream()
+ .collect(Collectors.toList()));
+ }
+ }
+ }
+ if (value instanceof Publication) {
+
+ } else if (value instanceof Dataset) {
+
+ } else if (value instanceof OtherResearchProduct) {
+
+ } else if (value instanceof Software) {
+
+ }
+
+ }
+
+ } else if (value instanceof Relation) {
+ Relation r = (Relation) value;
+
+ Optional validationDate = doCleanDate(r.getValidationDate());
+ if (validationDate.isPresent()) {
+ r.setValidationDate(validationDate.get());
+ r.setValidated(true);
+ } else {
+ r.setValidationDate(null);
+ r.setValidated(false);
+ }
+ }
+
+ return value;
+ }
+
+ private static Author cleanupAuthor(Author author) {
+ if (StringUtils.isNotBlank(author.getFullname())) {
+ author
+ .setFullname(
+ author
+ .getFullname()
+ .replaceAll(NAME_CLEANING_REGEX, " ")
+ .replace("\"", "\\\""));
+ }
+ if (StringUtils.isNotBlank(author.getName())) {
+ author
+ .setName(
+ author
+ .getName()
+ .replaceAll(NAME_CLEANING_REGEX, " ")
+ .replace("\"", "\\\""));
+ }
+ if (StringUtils.isNotBlank(author.getSurname())) {
+ author
+ .setSurname(
+ author
+ .getSurname()
+ .replaceAll(NAME_CLEANING_REGEX, " ")
+ .replace("\"", "\\\""));
+ }
+
+ return author;
+ }
+
+ private static Optional cleanDateField(Field dateofacceptance) {
+ return Optional
+ .ofNullable(dateofacceptance)
+ .map(Field::getValue)
+ .map(GraphCleaningFunctions::cleanDate)
+ .filter(Objects::nonNull);
+ }
+
+ protected static Optional doCleanDate(String date) {
+ return Optional.ofNullable(cleanDate(date));
+ }
+
+ public static String cleanDate(final String inputDate) {
+
+ if (StringUtils.isBlank(inputDate)) {
+ return null;
+ }
+
+ try {
+ final LocalDate date = DateParserUtils
+ .parseDate(inputDate.trim())
+ .toInstant()
+ .atZone(ZoneId.systemDefault())
+ .toLocalDate();
+ return DateTimeFormatter.ofPattern(ModelSupport.DATE_FORMAT).format(date);
+ } catch (DateTimeParseException e) {
+ return null;
+ }
+ }
+
+ // HELPERS
+
+ private static boolean isValidAuthorName(Author a) {
+ return StringUtils.isNotBlank(a.getFullname()) &&
+ StringUtils.isNotBlank(a.getFullname().replaceAll("[\\W]", "")) &&
+ !INVALID_AUTHOR_NAMES.contains(StringUtils.lowerCase(a.getFullname()).trim()) &&
+ !Stream
+ .of(a.getFullname(), a.getName(), a.getSurname())
+ .filter(StringUtils::isNotBlank)
+ .collect(Collectors.joining(""))
+ .toLowerCase()
+ .matches(INVALID_AUTHOR_REGEX);
+ }
+
+ private static boolean urlFilter(String u) {
+ try {
+ final URL url = new URL(u);
+ if (StringUtils.isBlank(url.getPath()) || "/".equals(url.getPath())) {
+ return false;
+ }
+ if (INVALID_URL_HOSTS.contains(url.getHost())) {
+ return false;
+ }
+ return !INVALID_URLS.contains(url.toString());
+ } catch (MalformedURLException ex) {
+ return false;
+ }
+ }
+
+ private static List processPidCleaning(List pids) {
+ return pids
+ .stream()
+ .filter(Objects::nonNull)
+ .filter(sp -> StringUtils.isNotBlank(StringUtils.trim(sp.getValue())))
+ .filter(sp -> !PID_BLACKLIST.contains(sp.getValue().trim().toLowerCase()))
+ .filter(sp -> Objects.nonNull(sp.getQualifier()))
+ .filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
+ .map(PidCleaner::normalizePidValue)
+ .filter(CleaningFunctions::pidFilter)
+ .collect(Collectors.toList());
+ }
+
+ private static void fixVocabName(Qualifier q, String vocabularyName) {
+ if (Objects.nonNull(q) && StringUtils.isBlank(q.getSchemeid())) {
+ q.setSchemeid(vocabularyName);
+ q.setSchemename(vocabularyName);
+ }
+ }
+
+ private static AccessRight accessRight(String classid, String classname, String scheme) {
+ return OafMapperUtils
+ .accessRight(
+ classid, classname, scheme, scheme);
+ }
+
+ private static Qualifier qualifier(String classid, String classname, String scheme) {
+ return OafMapperUtils
+ .qualifier(
+ classid, classname, scheme, scheme);
+ }
+
+ protected static StructuredProperty cleanValue(StructuredProperty s) {
+ s.setValue(s.getValue().replaceAll(CLEANING_REGEX, " "));
+ return s;
+ }
+
+ protected static Subject cleanValue(Subject s) {
+ s.setValue(s.getValue().replaceAll(CLEANING_REGEX, " "));
+ return s;
+ }
+
+ protected static Field cleanValue(Field s) {
+ s.setValue(s.getValue().replaceAll(CLEANING_REGEX, " "));
+ return s;
+ }
+
+ public static OafEntity applyCoarVocabularies(OafEntity entity, VocabularyGroup vocs) {
+
+ if (entity instanceof Result) {
+ final Result result = (Result) entity;
+
+ Optional
+ .ofNullable(result.getInstance())
+ .ifPresent(
+ instances -> instances
+ .forEach(
+ instance -> {
+ if (Objects.isNull(instance.getInstanceTypeMapping())) {
+ List mapping = Lists.newArrayList();
+ mapping
+ .add(
+ OafMapperUtils
+ .instanceTypeMapping(
+ instance.getInstancetype().getClassname(),
+ OPENAIRE_COAR_RESOURCE_TYPES_3_1));
+ instance.setInstanceTypeMapping(mapping);
+ }
+ Optional optionalItm = instance
+ .getInstanceTypeMapping()
+ .stream()
+ .filter(GraphCleaningFunctions::originalResourceType)
+ .findFirst();
+ if (optionalItm.isPresent()) {
+ InstanceTypeMapping coarItm = optionalItm.get();
+ Optional
+ .ofNullable(
+ vocs
+ .lookupTermBySynonym(
+ OPENAIRE_COAR_RESOURCE_TYPES_3_1, coarItm.getOriginalType()))
+ .ifPresent(type -> {
+ coarItm.setTypeCode(type.getClassid());
+ coarItm.setTypeLabel(type.getClassname());
+ });
+ final List mappings = Lists.newArrayList();
+ if (vocs.vocabularyExists(OPENAIRE_USER_RESOURCE_TYPES)) {
+ Optional
+ .ofNullable(
+ vocs
+ .lookupTermBySynonym(
+ OPENAIRE_USER_RESOURCE_TYPES, coarItm.getTypeCode()))
+ .ifPresent(
+ type -> mappings
+ .add(
+ OafMapperUtils
+ .instanceTypeMapping(coarItm.getTypeCode(), type)));
+ }
+ if (!mappings.isEmpty()) {
+ instance.getInstanceTypeMapping().addAll(mappings);
+ }
+ }
+ }));
+ result.setMetaResourceType(getMetaResourceType(result.getInstance(), vocs));
+ }
+
+ return entity;
+ }
+
+ private static boolean originalResourceType(InstanceTypeMapping itm) {
+ return StringUtils.isNotBlank(itm.getOriginalType()) &&
+ OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName()) &&
+ StringUtils.isBlank(itm.getTypeCode()) &&
+ StringUtils.isBlank(itm.getTypeLabel());
+ }
+
+ private static Qualifier getMetaResourceType(final List instances, final VocabularyGroup vocs) {
+ return Optional
+ .ofNullable(instances)
+ .map(ii -> {
+ if (vocs.vocabularyExists(OPENAIRE_META_RESOURCE_TYPE)) {
+ Optional itm = ii
+ .stream()
+ .filter(Objects::nonNull)
+ .flatMap(
+ i -> Optional
+ .ofNullable(i.getInstanceTypeMapping())
+ .map(Collection::stream)
+ .orElse(Stream.empty()))
+ .filter(t -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName()))
+ .findFirst();
+
+ if (!itm.isPresent() || Objects.isNull(itm.get().getTypeCode())) {
+ return null;
+ } else {
+ final String typeCode = itm.get().getTypeCode();
+ return Optional
+ .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_META_RESOURCE_TYPE, typeCode))
+ .orElseThrow(
+ () -> new IllegalStateException("unable to find a synonym for '" + typeCode + "' in " +
+ OPENAIRE_META_RESOURCE_TYPE));
+ }
+ } else {
+ throw new IllegalStateException("vocabulary '" + OPENAIRE_META_RESOURCE_TYPE + "' not available");
+ }
+ })
+ .orElse(null);
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GridCleaningRule.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GridCleaningRule.java
new file mode 100644
index 000000000..37ab91dd5
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GridCleaningRule.java
@@ -0,0 +1,24 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class GridCleaningRule {
+
+ public static final Pattern PATTERN = Pattern.compile("(?\\d{4,6}\\.[0-9a-z]{1,2})");
+
+ public static String clean(String grid) {
+ String s = grid
+ .replaceAll("\\s", "")
+ .toLowerCase();
+
+ Matcher m = PATTERN.matcher(s);
+ if (m.find()) {
+ return "grid." + m.group("grid");
+ }
+
+ return "";
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ISNICleaningRule.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ISNICleaningRule.java
new file mode 100644
index 000000000..bcd8279cc
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ISNICleaningRule.java
@@ -0,0 +1,21 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+// https://www.wikidata.org/wiki/Property:P213
+public class ISNICleaningRule {
+
+ public static final Pattern PATTERN = Pattern.compile("([0]{4}) ?([0-9]{4}) ?([0-9]{4}) ?([0-9]{3}[0-9X])");
+
+ public static String clean(final String isni) {
+
+ Matcher m = PATTERN.matcher(isni);
+ if (m.find()) {
+ return String.join("", m.group(1), m.group(2), m.group(3), m.group(4));
+ } else {
+ return "";
+ }
+ }
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java
new file mode 100644
index 000000000..4cecd0895
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java
@@ -0,0 +1,538 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
+
+import java.sql.Array;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+
+import eu.dnetlib.dhp.schema.common.AccessRightComparator;
+import eu.dnetlib.dhp.schema.common.ModelSupport;
+import eu.dnetlib.dhp.schema.oaf.*;
+
+public class OafMapperUtils {
+
+ private OafMapperUtils() {
+ }
+
+ public static Oaf merge(final Oaf left, final Oaf right) {
+ if (ModelSupport.isSubClass(left, OafEntity.class)) {
+ return mergeEntities((OafEntity) left, (OafEntity) right);
+ } else if (ModelSupport.isSubClass(left, Relation.class)) {
+ ((Relation) left).mergeFrom((Relation) right);
+ } else {
+ throw new IllegalArgumentException("invalid Oaf type:" + left.getClass().getCanonicalName());
+ }
+ return left;
+ }
+
+ public static OafEntity mergeEntities(OafEntity left, OafEntity right) {
+ if (ModelSupport.isSubClass(left, Result.class)) {
+ return mergeResults((Result) left, (Result) right);
+ } else if (ModelSupport.isSubClass(left, Datasource.class)) {
+ left.mergeFrom(right);
+ } else if (ModelSupport.isSubClass(left, Organization.class)) {
+ left.mergeFrom(right);
+ } else if (ModelSupport.isSubClass(left, Project.class)) {
+ left.mergeFrom(right);
+ } else {
+ throw new IllegalArgumentException("invalid OafEntity subtype:" + left.getClass().getCanonicalName());
+ }
+ return left;
+ }
+
+ public static Result mergeResults(Result left, Result right) {
+
+ final boolean leftFromDelegatedAuthority = isFromDelegatedAuthority(left);
+ final boolean rightFromDelegatedAuthority = isFromDelegatedAuthority(right);
+
+ if (leftFromDelegatedAuthority && !rightFromDelegatedAuthority) {
+ return left;
+ }
+ if (!leftFromDelegatedAuthority && rightFromDelegatedAuthority) {
+ return right;
+ }
+
+ if (new ResultTypeComparator().compare(left, right) < 0) {
+ left.mergeFrom(right);
+ return left;
+ } else {
+ right.mergeFrom(left);
+ return right;
+ }
+ }
+
+ private static boolean isFromDelegatedAuthority(Result r) {
+ return Optional
+ .ofNullable(r.getInstance())
+ .map(
+ instance -> instance
+ .stream()
+ .filter(i -> Objects.nonNull(i.getCollectedfrom()))
+ .map(i -> i.getCollectedfrom().getKey())
+ .anyMatch(cfId -> IdentifierFactory.delegatedAuthorityDatasourceIds().contains(cfId)))
+ .orElse(false);
+ }
+
+ public static KeyValue keyValue(final String k, final String v) {
+ final KeyValue kv = new KeyValue();
+ kv.setKey(k);
+ kv.setValue(v);
+ return kv;
+ }
+
+ public static List listKeyValues(final String... s) {
+ if (s.length % 2 > 0) {
+ throw new IllegalArgumentException("Invalid number of parameters (k,v,k,v,....)");
+ }
+
+ final List list = new ArrayList<>();
+ for (int i = 0; i < s.length; i += 2) {
+ list.add(keyValue(s[i], s[i + 1]));
+ }
+ return list;
+ }
+
+ public static Field field(final T value, final DataInfo info) {
+ if (value == null || StringUtils.isBlank(value.toString())) {
+ return null;
+ }
+
+ final Field field = new Field<>();
+ field.setValue(value);
+ field.setDataInfo(info);
+ return field;
+ }
+
+ public static List> listFields(final DataInfo info, final String... values) {
+ return Arrays
+ .stream(values)
+ .map(v -> field(v, info))
+ .filter(Objects::nonNull)
+ .filter(distinctByKey(Field::getValue))
+ .collect(Collectors.toList());
+ }
+
+ public static List listValues(Array values) throws SQLException {
+ if (Objects.isNull(values)) {
+ return null;
+ }
+ return Arrays
+ .stream((T[]) values.getArray())
+ .filter(Objects::nonNull)
+ .distinct()
+ .collect(Collectors.toList());
+ }
+
+ public static List> listFields(final DataInfo info, final List values) {
+ return values
+ .stream()
+ .map(v -> field(v, info))
+ .filter(Objects::nonNull)
+ .filter(distinctByKey(Field::getValue))
+ .collect(Collectors.toList());
+ }
+
+ public static InstanceTypeMapping instanceTypeMapping(String originalType, String code, String label,
+ String vocabularyName) {
+ final InstanceTypeMapping m = new InstanceTypeMapping();
+ m.setVocabularyName(vocabularyName);
+ m.setOriginalType(originalType);
+ m.setTypeCode(code);
+ m.setTypeLabel(label);
+ return m;
+ }
+
+ public static InstanceTypeMapping instanceTypeMapping(String originalType, Qualifier term) {
+ return instanceTypeMapping(originalType, term.getClassid(), term.getClassname(), term.getSchemeid());
+ }
+
+ public static InstanceTypeMapping instanceTypeMapping(String originalType) {
+ return instanceTypeMapping(originalType, null, null, null);
+ }
+
+ public static InstanceTypeMapping instanceTypeMapping(String originalType, String vocabularyName) {
+ return instanceTypeMapping(originalType, null, null, vocabularyName);
+ }
+
+ public static Qualifier unknown(final String schemeid, final String schemename) {
+ return qualifier(UNKNOWN, "Unknown", schemeid, schemename);
+ }
+
+ public static AccessRight accessRight(
+ final String classid,
+ final String classname,
+ final String schemeid,
+ final String schemename) {
+ return accessRight(classid, classname, schemeid, schemename, null);
+ }
+
+ public static AccessRight accessRight(
+ final String classid,
+ final String classname,
+ final String schemeid,
+ final String schemename,
+ final OpenAccessRoute openAccessRoute) {
+ final AccessRight accessRight = new AccessRight();
+ accessRight.setClassid(classid);
+ accessRight.setClassname(classname);
+ accessRight.setSchemeid(schemeid);
+ accessRight.setSchemename(schemename);
+ accessRight.setOpenAccessRoute(openAccessRoute);
+ return accessRight;
+ }
+
+ public static Qualifier qualifier(
+ final String classid,
+ final String classname,
+ final String schemeid,
+ final String schemename) {
+ final Qualifier q = new Qualifier();
+ q.setClassid(classid);
+ q.setClassname(classname);
+ q.setSchemeid(schemeid);
+ q.setSchemename(schemename);
+ return q;
+ }
+
+ public static Qualifier qualifier(final Qualifier qualifier) {
+ final Qualifier q = new Qualifier();
+ q.setClassid(qualifier.getClassid());
+ q.setClassname(qualifier.getClassname());
+ q.setSchemeid(qualifier.getSchemeid());
+ q.setSchemename(qualifier.getSchemename());
+ return q;
+ }
+
+ public static Subject subject(
+ final String value,
+ final String classid,
+ final String classname,
+ final String schemeid,
+ final String schemename,
+ final DataInfo dataInfo) {
+
+ return subject(value, qualifier(classid, classname, schemeid, schemename), dataInfo);
+ }
+
+ public static StructuredProperty structuredProperty(
+ final String value,
+ final String classid,
+ final String classname,
+ final String schemeid,
+ final String schemename,
+ final DataInfo dataInfo) {
+
+ return structuredProperty(value, qualifier(classid, classname, schemeid, schemename), dataInfo);
+ }
+
+ public static Subject subject(
+ final String value,
+ final Qualifier qualifier,
+ final DataInfo dataInfo) {
+ if (value == null) {
+ return null;
+ }
+ final Subject s = new Subject();
+ s.setValue(value);
+ s.setQualifier(qualifier);
+ s.setDataInfo(dataInfo);
+ return s;
+ }
+
+ public static StructuredProperty structuredProperty(
+ final String value,
+ final Qualifier qualifier,
+ final DataInfo dataInfo) {
+ if (value == null) {
+ return null;
+ }
+ final StructuredProperty sp = new StructuredProperty();
+ sp.setValue(value);
+ sp.setQualifier(qualifier);
+ sp.setDataInfo(dataInfo);
+ return sp;
+ }
+
+ public static ExtraInfo extraInfo(
+ final String name,
+ final String value,
+ final String typology,
+ final String provenance,
+ final String trust) {
+ final ExtraInfo info = new ExtraInfo();
+ info.setName(name);
+ info.setValue(value);
+ info.setTypology(typology);
+ info.setProvenance(provenance);
+ info.setTrust(trust);
+ return info;
+ }
+
+ public static OAIProvenance oaiIProvenance(
+ final String identifier,
+ final String baseURL,
+ final String metadataNamespace,
+ final Boolean altered,
+ final String datestamp,
+ final String harvestDate) {
+
+ final OriginDescription desc = new OriginDescription();
+ desc.setIdentifier(identifier);
+ desc.setBaseURL(baseURL);
+ desc.setMetadataNamespace(metadataNamespace);
+ desc.setAltered(altered);
+ desc.setDatestamp(datestamp);
+ desc.setHarvestDate(harvestDate);
+
+ final OAIProvenance p = new OAIProvenance();
+ p.setOriginDescription(desc);
+
+ return p;
+ }
+
+ public static Journal journal(
+ final String name,
+ final String issnPrinted,
+ final String issnOnline,
+ final String issnLinking,
+ final DataInfo dataInfo) {
+
+ return hasIssn(issnPrinted, issnOnline, issnLinking) ? journal(
+ name,
+ issnPrinted,
+ issnOnline,
+ issnLinking,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ dataInfo) : null;
+ }
+
+ public static Journal journal(
+ final String name,
+ final String issnPrinted,
+ final String issnOnline,
+ final String issnLinking,
+ final String ep,
+ final String iss,
+ final String sp,
+ final String vol,
+ final String edition,
+ final String conferenceplace,
+ final String conferencedate,
+ final DataInfo dataInfo) {
+
+ if (StringUtils.isNotBlank(name) || hasIssn(issnPrinted, issnOnline, issnLinking)) {
+ final Journal j = new Journal();
+ j.setName(name);
+ j.setIssnPrinted(issnPrinted);
+ j.setIssnOnline(issnOnline);
+ j.setIssnLinking(issnLinking);
+ j.setEp(ep);
+ j.setIss(iss);
+ j.setSp(sp);
+ j.setVol(vol);
+ j.setEdition(edition);
+ j.setConferenceplace(conferenceplace);
+ j.setConferencedate(conferencedate);
+ j.setDataInfo(dataInfo);
+ return j;
+ } else {
+ return null;
+ }
+ }
+
+ private static boolean hasIssn(String issnPrinted, String issnOnline, String issnLinking) {
+ return StringUtils.isNotBlank(issnPrinted)
+ || StringUtils.isNotBlank(issnOnline)
+ || StringUtils.isNotBlank(issnLinking);
+ }
+
+ public static DataInfo dataInfo(
+ final Boolean deletedbyinference,
+ final String inferenceprovenance,
+ final Boolean inferred,
+ final Boolean invisible,
+ final Qualifier provenanceaction,
+ final String trust) {
+ final DataInfo d = new DataInfo();
+ d.setDeletedbyinference(deletedbyinference);
+ d.setInferenceprovenance(inferenceprovenance);
+ d.setInferred(inferred);
+ d.setInvisible(invisible);
+ d.setProvenanceaction(provenanceaction);
+ d.setTrust(trust);
+ return d;
+ }
+
+ public static String createOpenaireId(
+ final int prefix,
+ final String originalId,
+ final boolean to_md5) {
+ if (StringUtils.isBlank(originalId)) {
+ return null;
+ } else if (to_md5) {
+ final String nsPrefix = StringUtils.substringBefore(originalId, "::");
+ final String rest = StringUtils.substringAfter(originalId, "::");
+ return String.format("%s|%s::%s", prefix, nsPrefix, IdentifierFactory.md5(rest));
+ } else {
+ return String.format("%s|%s", prefix, originalId);
+ }
+ }
+
+ public static String createOpenaireId(
+ final String type,
+ final String originalId,
+ final boolean to_md5) {
+ switch (type) {
+ case "datasource":
+ return createOpenaireId(10, originalId, to_md5);
+ case "organization":
+ return createOpenaireId(20, originalId, to_md5);
+ case "person":
+ return createOpenaireId(30, originalId, to_md5);
+ case "project":
+ return createOpenaireId(40, originalId, to_md5);
+ default:
+ return createOpenaireId(50, originalId, to_md5);
+ }
+ }
+
+ public static String asString(final Object o) {
+ return o == null ? "" : o.toString();
+ }
+
+ public static Predicate distinctByKey(
+ final Function keyExtractor) {
+ final Map seen = new ConcurrentHashMap<>();
+ return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
+ }
+
+ public static Qualifier createBestAccessRights(final List instanceList) {
+ return getBestAccessRights(instanceList);
+ }
+
+ protected static Qualifier getBestAccessRights(final List instanceList) {
+ if (instanceList != null) {
+ final Optional min = instanceList
+ .stream()
+ .map(Instance::getAccessright)
+ .min(new AccessRightComparator<>());
+
+ final Qualifier rights = min.map(OafMapperUtils::qualifier).orElseGet(Qualifier::new);
+
+ if (StringUtils.isBlank(rights.getClassid())) {
+ rights.setClassid(UNKNOWN);
+ }
+ if (StringUtils.isBlank(rights.getClassname())
+ || UNKNOWN.equalsIgnoreCase(rights.getClassname())) {
+ rights.setClassname(NOT_AVAILABLE);
+ }
+ if (StringUtils.isBlank(rights.getSchemeid())) {
+ rights.setSchemeid(DNET_ACCESS_MODES);
+ }
+ if (StringUtils.isBlank(rights.getSchemename())) {
+ rights.setSchemename(DNET_ACCESS_MODES);
+ }
+
+ return rights;
+ }
+ return null;
+ }
+
+ public static KeyValue newKeyValueInstance(String key, String value, DataInfo dataInfo) {
+ KeyValue kv = new KeyValue();
+ kv.setDataInfo(dataInfo);
+ kv.setKey(key);
+ kv.setValue(value);
+ return kv;
+ }
+
+ public static Measure newMeasureInstance(String id, String value, String key, DataInfo dataInfo) {
+ Measure m = new Measure();
+ m.setId(id);
+ m.setUnit(Arrays.asList(newKeyValueInstance(key, value, dataInfo)));
+ return m;
+ }
+
+ public static Relation getRelation(final String source,
+ final String target,
+ final String relType,
+ final String subRelType,
+ final String relClass,
+ final OafEntity entity) {
+ return getRelation(source, target, relType, subRelType, relClass, entity, null);
+ }
+
+ public static Relation getRelation(final String source,
+ final String target,
+ final String relType,
+ final String subRelType,
+ final String relClass,
+ final OafEntity entity,
+ final String validationDate) {
+ return getRelation(
+ source, target, relType, subRelType, relClass, entity.getCollectedfrom(), entity.getDataInfo(),
+ entity.getLastupdatetimestamp(), validationDate, null);
+ }
+
+ public static Relation getRelation(final String source,
+ final String target,
+ final String relType,
+ final String subRelType,
+ final String relClass,
+ final List collectedfrom,
+ final DataInfo dataInfo,
+ final Long lastupdatetimestamp) {
+ return getRelation(
+ source, target, relType, subRelType, relClass, collectedfrom, dataInfo, lastupdatetimestamp, null, null);
+ }
+
+ public static Relation getRelation(final String source,
+ final String target,
+ final String relType,
+ final String subRelType,
+ final String relClass,
+ final List collectedfrom,
+ final DataInfo dataInfo,
+ final Long lastupdatetimestamp,
+ final String validationDate,
+ final List properties) {
+ final Relation rel = new Relation();
+ rel.setRelType(relType);
+ rel.setSubRelType(subRelType);
+ rel.setRelClass(relClass);
+ rel.setSource(source);
+ rel.setTarget(target);
+ rel.setCollectedfrom(collectedfrom);
+ rel.setDataInfo(dataInfo);
+ rel.setLastupdatetimestamp(lastupdatetimestamp);
+ rel.setValidated(StringUtils.isNotBlank(validationDate));
+ rel.setValidationDate(StringUtils.isNotBlank(validationDate) ? validationDate : null);
+ rel.setProperties(properties);
+ return rel;
+ }
+
+ public static String getProvenance(DataInfo dataInfo) {
+ return Optional
+ .ofNullable(dataInfo)
+ .map(
+ d -> Optional
+ .ofNullable(d.getProvenanceaction())
+ .map(Qualifier::getClassid)
+ .orElse(""))
+ .orElse("");
+ }
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PICCleaningRule.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PICCleaningRule.java
new file mode 100644
index 000000000..a2213ed9f
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PICCleaningRule.java
@@ -0,0 +1,21 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PICCleaningRule {
+
+ public static final Pattern PATTERN = Pattern.compile("\\d{9}");
+
+ public static String clean(final String pic) {
+
+ Matcher m = PATTERN.matcher(pic);
+ if (m.find()) {
+ return m.group();
+ } else {
+ return "";
+ }
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PidCleaner.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PidCleaner.java
new file mode 100644
index 000000000..114c2b3af
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PidCleaner.java
@@ -0,0 +1,62 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import java.util.Optional;
+
+import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
+
+public class PidCleaner {
+
+ /**
+ * Utility method that normalises PID values on a per-type basis.
+ * @param pid the PID whose value will be normalised.
+ * @return the PID containing the normalised value.
+ */
+ public static StructuredProperty normalizePidValue(StructuredProperty pid) {
+ pid
+ .setValue(
+ normalizePidValue(
+ pid.getQualifier().getClassid(),
+ pid.getValue()));
+
+ return pid;
+ }
+
+ public static String normalizePidValue(String pidType, String pidValue) {
+ String value = Optional
+ .ofNullable(pidValue)
+ .map(String::trim)
+ .orElseThrow(() -> new IllegalArgumentException("PID value cannot be empty"));
+
+ switch (pidType) {
+
+ // TODO add cleaning for more PID types as needed
+
+ // Result
+ case "doi":
+ return DoiCleaningRule.clean(value);
+ case "pmid":
+ return PmidCleaningRule.clean(value);
+ case "pmc":
+ return PmcCleaningRule.clean(value);
+ case "handle":
+ case "arXiv":
+ return value;
+
+ // Organization
+ case "GRID":
+ return GridCleaningRule.clean(value);
+ case "ISNI":
+ return ISNICleaningRule.clean(value);
+ case "ROR":
+ return RorCleaningRule.clean(value);
+ case "PIC":
+ return PICCleaningRule.clean(value);
+ case "FundRef":
+ return FundRefCleaningRule.clean(value);
+ default:
+ return value;
+ }
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PmcCleaningRule.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PmcCleaningRule.java
new file mode 100644
index 000000000..903041d43
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PmcCleaningRule.java
@@ -0,0 +1,24 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PmcCleaningRule {
+
+ public static final Pattern PATTERN = Pattern.compile("PMC\\d{1,8}");
+
+ public static String clean(String pmc) {
+ String s = pmc
+ .replaceAll("\\s", "")
+ .toUpperCase();
+
+ final Matcher m = PATTERN.matcher(s);
+
+ if (m.find()) {
+ return m.group();
+ }
+ return "";
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRule.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRule.java
new file mode 100644
index 000000000..c0c451b88
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRule.java
@@ -0,0 +1,25 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+// https://researchguides.stevens.edu/c.php?g=442331&p=6577176
+public class PmidCleaningRule {
+
+ public static final Pattern PATTERN = Pattern.compile("0*(\\d{1,8})");
+
+ public static String clean(String pmid) {
+ String s = pmid
+ .toLowerCase()
+ .replaceAll("\\s", "");
+
+ final Matcher m = PATTERN.matcher(s);
+
+ if (m.find()) {
+ return m.group(1);
+ }
+ return "";
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/RorCleaningRule.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/RorCleaningRule.java
new file mode 100644
index 000000000..f6685f19d
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/RorCleaningRule.java
@@ -0,0 +1,27 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+// https://ror.readme.io/docs/ror-identifier-pattern
+public class RorCleaningRule {
+
+ public static final String ROR_PREFIX = "https://ror.org/";
+
+ private static final Pattern PATTERN = Pattern.compile("(?0[a-hj-km-np-tv-z|0-9]{6}[0-9]{2})");
+
+ public static String clean(String ror) {
+ String s = ror
+ .replaceAll("\\s", "")
+ .toLowerCase();
+
+ Matcher m = PATTERN.matcher(s);
+
+ if (m.find()) {
+ return ROR_PREFIX + m.group("ror");
+ }
+ return "";
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/SubjectProvenanceComparator.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/SubjectProvenanceComparator.java
new file mode 100644
index 000000000..f4e3c8841
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/SubjectProvenanceComparator.java
@@ -0,0 +1,46 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.getProvenance;
+import static org.apache.commons.lang3.StringUtils.isBlank;
+
+import java.util.Comparator;
+
+import eu.dnetlib.dhp.schema.oaf.Subject;
+
+public class SubjectProvenanceComparator implements Comparator {
+
+ @Override
+ public int compare(Subject left, Subject right) {
+
+ String lProv = getProvenance(left.getDataInfo());
+ String rProv = getProvenance(right.getDataInfo());
+
+ if (isBlank(lProv) && isBlank(rProv))
+ return 0;
+ if (isBlank(lProv))
+ return 1;
+ if (isBlank(rProv))
+ return -1;
+ if (lProv.equals(rProv))
+ return 0;
+ if (lProv.toLowerCase().contains("crosswalk"))
+ return -1;
+ if (rProv.toLowerCase().contains("crosswalk"))
+ return 1;
+ if (lProv.toLowerCase().contains("user"))
+ return -1;
+ if (rProv.toLowerCase().contains("user"))
+ return 1;
+ if (lProv.toLowerCase().contains("propagation"))
+ return -1;
+ if (rProv.toLowerCase().contains("propagation"))
+ return 1;
+ if (lProv.toLowerCase().contains("iis"))
+ return -1;
+ if (rProv.toLowerCase().contains("iis"))
+ return 1;
+
+ return 0;
+ }
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java
index 8872174a5..e10d0c500 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java
@@ -1,26 +1,44 @@
package eu.dnetlib.dhp.utils;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import java.io.*;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
-import java.util.List;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
+import java.util.*;
+import java.util.stream.Collectors;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Base64OutputStream;
import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SaveMode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Maps;
import com.jayway.jsonpath.JsonPath;
+import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
+import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
import net.minidev.json.JSONArray;
import scala.collection.JavaConverters;
import scala.collection.Seq;
public class DHPUtils {
+ private static final Logger log = LoggerFactory.getLogger(DHPUtils.class);
+
+ private DHPUtils() {
+ }
+
public static Seq toSeq(List list) {
return JavaConverters.asScalaIteratorConverter(list.iterator()).asScala().toSeq();
}
@@ -31,40 +49,64 @@ public class DHPUtils {
md.update(s.getBytes(StandardCharsets.UTF_8));
return new String(Hex.encodeHex(md.digest()));
} catch (final Exception e) {
- System.err.println("Error creating id");
+ log.error("Error creating id from {}", s);
return null;
}
}
+ /**
+ * Retrieves from the metadata store manager application the list of paths associated with mdstores characterized
+ * by he given format, layout, interpretation
+ * @param mdstoreManagerUrl the URL of the mdstore manager service
+ * @param format the mdstore format
+ * @param layout the mdstore layout
+ * @param interpretation the mdstore interpretation
+ * @param includeEmpty include Empty mdstores
+ * @return the set of hdfs paths
+ * @throws IOException in case of HTTP communication issues
+ */
+ public static Set mdstorePaths(final String mdstoreManagerUrl,
+ final String format,
+ final String layout,
+ final String interpretation,
+ boolean includeEmpty) throws IOException {
+ final String url = mdstoreManagerUrl + "/mdstores/";
+ final ObjectMapper objectMapper = new ObjectMapper();
+
+ final HttpGet req = new HttpGet(url);
+
+ log.info("MDStoreManager request: {}", req);
+
+ try (final CloseableHttpClient client = HttpClients.createDefault()) {
+ try (final CloseableHttpResponse response = client.execute(req)) {
+ final String json = IOUtils.toString(response.getEntity().getContent());
+
+ log.info("MDStoreManager response: {}", json);
+
+ final MDStoreWithInfo[] mdstores = objectMapper.readValue(json, MDStoreWithInfo[].class);
+ return Arrays
+ .stream(mdstores)
+ .filter(md -> md.getFormat().equalsIgnoreCase(format))
+ .filter(md -> md.getLayout().equalsIgnoreCase(layout))
+ .filter(md -> md.getInterpretation().equalsIgnoreCase(interpretation))
+ .filter(md -> StringUtils.isNotBlank(md.getHdfsPath()))
+ .filter(md -> StringUtils.isNotBlank(md.getCurrentVersion()))
+ .filter(md -> includeEmpty || md.getSize() > 0)
+ .map(md -> md.getHdfsPath() + "/" + md.getCurrentVersion() + "/store")
+ .collect(Collectors.toSet());
+ }
+ }
+ }
+
public static String generateIdentifier(final String originalId, final String nsPrefix) {
return String.format("%s::%s", nsPrefix, DHPUtils.md5(originalId));
}
- public static String compressString(final String input) {
- try (ByteArrayOutputStream out = new ByteArrayOutputStream();
- Base64OutputStream b64os = new Base64OutputStream(out)) {
- GZIPOutputStream gzip = new GZIPOutputStream(b64os);
- gzip.write(input.getBytes(StandardCharsets.UTF_8));
- gzip.close();
- return out.toString();
- } catch (Throwable e) {
- return null;
- }
- }
+ public static String generateUnresolvedIdentifier(final String pid, final String pidType) {
- public static String decompressString(final String input) {
- byte[] byteArray = Base64.decodeBase64(input.getBytes());
- int len;
- try (GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream((byteArray)));
- ByteArrayOutputStream bos = new ByteArrayOutputStream(byteArray.length)) {
- byte[] buffer = new byte[1024];
- while ((len = gis.read(buffer)) != -1) {
- bos.write(buffer, 0, len);
- }
- return bos.toString();
- } catch (Exception e) {
- return null;
- }
+ final String cleanedPid = CleaningFunctions.normalizePidValue(pidType, pid);
+
+ return String.format("unresolved::%s::%s", cleanedPid, pidType.toLowerCase().trim());
}
public static String getJPathString(final String jsonPath, final String json) {
@@ -79,4 +121,72 @@ public class DHPUtils {
return "";
}
}
+
+ public static final ObjectMapper MAPPER = new ObjectMapper();
+
+ public static void writeHdfsFile(final Configuration conf, final String content, final String path)
+ throws IOException {
+
+ log.info("writing file {}, size {}", path, content.length());
+ try (FileSystem fs = FileSystem.get(conf);
+ BufferedOutputStream os = new BufferedOutputStream(fs.create(new Path(path)))) {
+ os.write(content.getBytes(StandardCharsets.UTF_8));
+ os.flush();
+ }
+ }
+
+ public static String readHdfsFile(Configuration conf, String path) throws IOException {
+ log.info("reading file {}", path);
+
+ try (FileSystem fs = FileSystem.get(conf)) {
+ final Path p = new Path(path);
+ if (!fs.exists(p)) {
+ throw new FileNotFoundException(path);
+ }
+ return IOUtils.toString(fs.open(p));
+ }
+ }
+
+ public static T readHdfsFileAs(Configuration conf, String path, Class clazz) throws IOException {
+ return MAPPER.readValue(readHdfsFile(conf, path), clazz);
+ }
+
+ public static void saveDataset(final Dataset mdstore, final String targetPath) {
+ log.info("saving dataset in: {}", targetPath);
+ mdstore
+ .write()
+ .mode(SaveMode.Overwrite)
+ .format("parquet")
+ .save(targetPath);
+ }
+
+ public static Configuration getHadoopConfiguration(String nameNode) {
+ // ====== Init HDFS File System Object
+ Configuration conf = new Configuration();
+ // Set FileSystem URI
+ conf.set("fs.defaultFS", nameNode);
+ // Because of Maven
+ conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
+ conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
+
+ System.setProperty("hadoop.home.dir", "/");
+ return conf;
+ }
+
+ public static void populateOOZIEEnv(final Map report) throws IOException {
+ File file = new File(System.getProperty("oozie.action.output.properties"));
+ Properties props = new Properties();
+ report.forEach((k, v) -> props.setProperty(k, v));
+
+ try (OutputStream os = new FileOutputStream(file)) {
+ props.store(os, "");
+ }
+ }
+
+ public static void populateOOZIEEnv(final String paramName, String value) throws IOException {
+ Map report = Maps.newHashMap();
+ report.put(paramName, value);
+
+ populateOOZIEEnv(report);
+ }
}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/ISLookupClientFactory.java b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/ISLookupClientFactory.java
index 9552eb2b3..8ae0bb5c3 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/ISLookupClientFactory.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/ISLookupClientFactory.java
@@ -1,11 +1,11 @@
package eu.dnetlib.dhp.utils;
-import java.util.Map;
-
-import javax.xml.ws.BindingProvider;
-
+import org.apache.cxf.endpoint.Client;
+import org.apache.cxf.frontend.ClientProxy;
import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
+import org.apache.cxf.transport.http.HTTPConduit;
+import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -15,8 +15,11 @@ public class ISLookupClientFactory {
private static final Logger log = LoggerFactory.getLogger(ISLookupClientFactory.class);
- private static int requestTimeout = 60000 * 10;
- private static int connectTimeout = 60000 * 10;
+ private static final int requestTimeout = 60000 * 10;
+ private static final int connectTimeout = 60000 * 10;
+
+ private ISLookupClientFactory() {
+ }
public static ISLookUpService getLookUpService(final String isLookupUrl) {
return getServiceStub(ISLookUpService.class, isLookupUrl);
@@ -24,27 +27,28 @@ public class ISLookupClientFactory {
@SuppressWarnings("unchecked")
private static T getServiceStub(final Class clazz, final String endpoint) {
- log.info(String.format("creating %s stub from %s", clazz.getName(), endpoint));
+ log.info("creating {} stub from {}", clazz.getName(), endpoint);
final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
jaxWsProxyFactory.setServiceClass(clazz);
jaxWsProxyFactory.setAddress(endpoint);
final T service = (T) jaxWsProxyFactory.create();
- if (service instanceof BindingProvider) {
+ Client client = ClientProxy.getClient(service);
+ if (client != null) {
+ HTTPConduit conduit = (HTTPConduit) client.getConduit();
+ HTTPClientPolicy policy = new HTTPClientPolicy();
+
log
.info(
- "setting timeouts for {} to requestTimeout: {}, connectTimeout: {}",
- BindingProvider.class.getName(), requestTimeout, connectTimeout);
+ "setting connectTimeout to {}, requestTimeout to {} for service {}",
+ connectTimeout,
+ requestTimeout,
+ clazz.getCanonicalName());
- Map requestContext = ((BindingProvider) service).getRequestContext();
-
- requestContext.put("com.sun.xml.internal.ws.request.timeout", requestTimeout);
- requestContext.put("com.sun.xml.internal.ws.connect.timeout", connectTimeout);
- requestContext.put("com.sun.xml.ws.request.timeout", requestTimeout);
- requestContext.put("com.sun.xml.ws.connect.timeout", connectTimeout);
- requestContext.put("javax.xml.ws.client.receiveTimeout", requestTimeout);
- requestContext.put("javax.xml.ws.client.connectionTimeout", connectTimeout);
+ policy.setConnectionTimeout(connectTimeout);
+ policy.setReceiveTimeout(requestTimeout);
+ conduit.setClient(policy);
}
return service;
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/AbstractExtensionFunction.java b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/AbstractExtensionFunction.java
index 9b00b908c..81f1b5142 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/AbstractExtensionFunction.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/AbstractExtensionFunction.java
@@ -10,7 +10,7 @@ import net.sf.saxon.trans.XPathException;
public abstract class AbstractExtensionFunction extends ExtensionFunctionDefinition {
- public static String DEFAULT_SAXON_EXT_NS_URI = "http://www.d-net.research-infrastructures.eu/saxon-extension";
+ public static final String DEFAULT_SAXON_EXT_NS_URI = "http://www.d-net.research-infrastructures.eu/saxon-extension";
public abstract String getName();
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/ExtractYear.java b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/ExtractYear.java
index c7e311b02..1ea2b9f46 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/ExtractYear.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/ExtractYear.java
@@ -26,7 +26,7 @@ public class ExtractYear extends AbstractExtensionFunction {
@Override
public Sequence doCall(XPathContext context, Sequence[] arguments) throws XPathException {
- if (arguments == null | arguments.length == 0) {
+ if (arguments == null || arguments.length == 0) {
return new StringValue("");
}
final Item item = arguments[0].head();
@@ -63,8 +63,7 @@ public class ExtractYear extends AbstractExtensionFunction {
for (String format : dateFormats) {
try {
c.setTime(new SimpleDateFormat(format).parse(s));
- String year = String.valueOf(c.get(Calendar.YEAR));
- return year;
+ return String.valueOf(c.get(Calendar.YEAR));
} catch (ParseException e) {
}
}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/NormalizeDate.java b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/NormalizeDate.java
index 1b5f3c40d..3e5def9b5 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/NormalizeDate.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/NormalizeDate.java
@@ -30,7 +30,7 @@ public class NormalizeDate extends AbstractExtensionFunction {
@Override
public Sequence doCall(XPathContext context, Sequence[] arguments) throws XPathException {
- if (arguments == null | arguments.length == 0) {
+ if (arguments == null || arguments.length == 0) {
return new StringValue(BLANK);
}
String s = arguments[0].head().getStringValue();
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/PickFirst.java b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/PickFirst.java
index 46ecafd0a..b46a415d8 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/PickFirst.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/PickFirst.java
@@ -1,6 +1,8 @@
package eu.dnetlib.dhp.utils.saxon;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
import org.apache.commons.lang3.StringUtils;
import net.sf.saxon.expr.XPathContext;
@@ -26,7 +28,8 @@ public class PickFirst extends AbstractExtensionFunction {
final String s1 = getValue(arguments[0]);
final String s2 = getValue(arguments[1]);
- return new StringValue(StringUtils.isNotBlank(s1) ? s1 : StringUtils.isNotBlank(s2) ? s2 : "");
+ final String value = isNotBlank(s1) ? s1 : isNotBlank(s2) ? s2 : "";
+ return new StringValue(value);
}
private String getValue(final Sequence arg) throws XPathException {
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/SaxonTransformerFactory.java b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/SaxonTransformerFactory.java
index b85d866f1..61049d2e1 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/SaxonTransformerFactory.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/SaxonTransformerFactory.java
@@ -12,6 +12,9 @@ import net.sf.saxon.TransformerFactoryImpl;
public class SaxonTransformerFactory {
+ private SaxonTransformerFactory() {
+ }
+
/**
* Creates the index record transformer from the given XSLT
*
diff --git a/dhp-common/src/main/java/eu/dnetlib/message/Message.java b/dhp-common/src/main/java/eu/dnetlib/message/Message.java
deleted file mode 100644
index fc1c38291..000000000
--- a/dhp-common/src/main/java/eu/dnetlib/message/Message.java
+++ /dev/null
@@ -1,76 +0,0 @@
-
-package eu.dnetlib.message;
-
-import java.io.IOException;
-import java.util.Map;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-public class Message {
-
- private String workflowId;
-
- private String jobName;
-
- private MessageType type;
-
- private Map body;
-
- public static Message fromJson(final String json) throws IOException {
- final ObjectMapper jsonMapper = new ObjectMapper();
- return jsonMapper.readValue(json, Message.class);
- }
-
- public Message() {
- }
-
- public Message(String workflowId, String jobName, MessageType type, Map body) {
- this.workflowId = workflowId;
- this.jobName = jobName;
- this.type = type;
- this.body = body;
- }
-
- public String getWorkflowId() {
- return workflowId;
- }
-
- public void setWorkflowId(String workflowId) {
- this.workflowId = workflowId;
- }
-
- public String getJobName() {
- return jobName;
- }
-
- public void setJobName(String jobName) {
- this.jobName = jobName;
- }
-
- public MessageType getType() {
- return type;
- }
-
- public void setType(MessageType type) {
- this.type = type;
- }
-
- public Map getBody() {
- return body;
- }
-
- public void setBody(Map body) {
- this.body = body;
- }
-
- @Override
- public String toString() {
- final ObjectMapper jsonMapper = new ObjectMapper();
- try {
- return jsonMapper.writeValueAsString(this);
- } catch (JsonProcessingException e) {
- return null;
- }
- }
-}
diff --git a/dhp-common/src/main/java/eu/dnetlib/message/MessageConsumer.java b/dhp-common/src/main/java/eu/dnetlib/message/MessageConsumer.java
deleted file mode 100644
index fb3f0bd95..000000000
--- a/dhp-common/src/main/java/eu/dnetlib/message/MessageConsumer.java
+++ /dev/null
@@ -1,47 +0,0 @@
-
-package eu.dnetlib.message;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import com.rabbitmq.client.AMQP;
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.DefaultConsumer;
-import com.rabbitmq.client.Envelope;
-
-public class MessageConsumer extends DefaultConsumer {
-
- final LinkedBlockingQueue queueMessages;
-
- /**
- * Constructs a new instance and records its association to the passed-in channel.
- *
- * @param channel the channel to which this consumer is attached
- * @param queueMessages
- */
- public MessageConsumer(Channel channel, LinkedBlockingQueue queueMessages) {
- super(channel);
- this.queueMessages = queueMessages;
- }
-
- @Override
- public void handleDelivery(
- String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body)
- throws IOException {
- final String json = new String(body, StandardCharsets.UTF_8);
- Message message = Message.fromJson(json);
- try {
- this.queueMessages.put(message);
- System.out.println("Receiving Message " + message);
- } catch (InterruptedException e) {
- if (message.getType() == MessageType.REPORT)
- throw new RuntimeException("Error on sending message");
- else {
- // TODO LOGGING EXCEPTION
- }
- } finally {
- getChannel().basicAck(envelope.getDeliveryTag(), false);
- }
- }
-}
diff --git a/dhp-common/src/main/java/eu/dnetlib/message/MessageManager.java b/dhp-common/src/main/java/eu/dnetlib/message/MessageManager.java
deleted file mode 100644
index 5ca79f3cc..000000000
--- a/dhp-common/src/main/java/eu/dnetlib/message/MessageManager.java
+++ /dev/null
@@ -1,136 +0,0 @@
-
-package eu.dnetlib.message;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeoutException;
-
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
-
-public class MessageManager {
-
- private final String messageHost;
-
- private final String username;
-
- private final String password;
-
- private Connection connection;
-
- private final Map channels = new HashMap<>();
-
- private boolean durable;
-
- private boolean autodelete;
-
- private final LinkedBlockingQueue queueMessages;
-
- public MessageManager(
- String messageHost,
- String username,
- String password,
- final LinkedBlockingQueue queueMessages) {
- this.queueMessages = queueMessages;
- this.messageHost = messageHost;
- this.username = username;
- this.password = password;
- }
-
- public MessageManager(
- String messageHost,
- String username,
- String password,
- boolean durable,
- boolean autodelete,
- final LinkedBlockingQueue queueMessages) {
- this.queueMessages = queueMessages;
- this.messageHost = messageHost;
- this.username = username;
- this.password = password;
-
- this.durable = durable;
- this.autodelete = autodelete;
- }
-
- private Connection createConnection() throws IOException, TimeoutException {
- ConnectionFactory factory = new ConnectionFactory();
- factory.setHost(this.messageHost);
- factory.setUsername(this.username);
- factory.setPassword(this.password);
- return factory.newConnection();
- }
-
- private Channel createChannel(
- final Connection connection,
- final String queueName,
- final boolean durable,
- final boolean autodelete)
- throws Exception {
- Map args = new HashMap<>();
- args.put("x-message-ttl", 10000);
- Channel channel = connection.createChannel();
- channel.queueDeclare(queueName, durable, false, this.autodelete, args);
- return channel;
- }
-
- private Channel getOrCreateChannel(final String queueName, boolean durable, boolean autodelete)
- throws Exception {
- if (channels.containsKey(queueName)) {
- return channels.get(queueName);
- }
-
- if (this.connection == null) {
- this.connection = createConnection();
- }
- channels.put(queueName, createChannel(this.connection, queueName, durable, autodelete));
- return channels.get(queueName);
- }
-
- public void close() throws IOException {
- channels
- .values()
- .forEach(
- ch -> {
- try {
- ch.close();
- } catch (Exception e) {
- // TODO LOG
- }
- });
-
- this.connection.close();
- }
-
- public boolean sendMessage(final Message message, String queueName) throws Exception {
- try {
- Channel channel = getOrCreateChannel(queueName, this.durable, this.autodelete);
- channel.basicPublish("", queueName, null, message.toString().getBytes());
- return true;
- } catch (Throwable e) {
- throw new RuntimeException(e);
- }
- }
-
- public boolean sendMessage(
- final Message message, String queueName, boolean durable_var, boolean autodelete_var)
- throws Exception {
- try {
- Channel channel = getOrCreateChannel(queueName, durable_var, autodelete_var);
- channel.basicPublish("", queueName, null, message.toString().getBytes());
- return true;
- } catch (Throwable e) {
- throw new RuntimeException(e);
- }
- }
-
- public void startConsumingMessage(
- final String queueName, final boolean durable, final boolean autodelete) throws Exception {
-
- Channel channel = createChannel(createConnection(), queueName, durable, autodelete);
- channel.basicConsume(queueName, false, new MessageConsumer(channel, queueMessages));
- }
-}
diff --git a/dhp-common/src/main/java/eu/dnetlib/message/MessageType.java b/dhp-common/src/main/java/eu/dnetlib/message/MessageType.java
deleted file mode 100644
index 72cbda252..000000000
--- a/dhp-common/src/main/java/eu/dnetlib/message/MessageType.java
+++ /dev/null
@@ -1,6 +0,0 @@
-
-package eu.dnetlib.message;
-
-public enum MessageType {
- ONGOING, REPORT
-}
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_maketar_parameters.json b/dhp-common/src/main/resources/eu/dnetlib/dhp/common/input_maketar_parameters.json
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_maketar_parameters.json
rename to dhp-common/src/main/resources/eu/dnetlib/dhp/common/input_maketar_parameters.json
diff --git a/dhp-common/src/main/resources/eu/dnetlib/dhp/common/name_particles.txt b/dhp-common/src/main/resources/eu/dnetlib/dhp/common/name_particles.txt
new file mode 100644
index 000000000..07cf06a98
--- /dev/null
+++ b/dhp-common/src/main/resources/eu/dnetlib/dhp/common/name_particles.txt
@@ -0,0 +1,8 @@
+van
+von
+der
+de
+dell
+sig
+mr
+mrs
\ No newline at end of file
diff --git a/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json b/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json
new file mode 100644
index 000000000..512878457
--- /dev/null
+++ b/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json
@@ -0,0 +1,38 @@
+[
+ {
+ "paramName": "issm",
+ "paramLongName": "isSparkSessionManaged",
+ "paramDescription": "when true will stop SparkSession after job execution",
+ "paramRequired": false
+ },
+ {
+ "paramName": "gin",
+ "paramLongName": "graphInputPath",
+ "paramDescription": "the input graph root path",
+ "paramRequired": true
+ },
+ {
+ "paramName": "cp",
+ "paramLongName": "checkpointPath",
+ "paramDescription": "checkpoint directory",
+ "paramRequired": true
+ },
+ {
+ "paramName": "out",
+ "paramLongName": "outputPath",
+ "paramDescription": "the output graph root path",
+ "paramRequired": true
+ },
+ {
+ "paramName": "fi",
+ "paramLongName": "filterInvisible",
+ "paramDescription": "if true filters out invisible entities",
+ "paramRequired": true
+ },
+ {
+ "paramName": "isu",
+ "paramLongName": "isLookupUrl",
+ "paramDescription": "url to the ISLookup Service",
+ "paramRequired": true
+ }
+]
\ No newline at end of file
diff --git a/dhp-common/src/main/resources/eu/dnetlib/dhp/oozie/run_sql_parameters.json b/dhp-common/src/main/resources/eu/dnetlib/dhp/oozie/run_sql_parameters.json
new file mode 100644
index 000000000..355f38e2f
--- /dev/null
+++ b/dhp-common/src/main/resources/eu/dnetlib/dhp/oozie/run_sql_parameters.json
@@ -0,0 +1,20 @@
+[
+ {
+ "paramName": "issm",
+ "paramLongName": "isSparkSessionManaged",
+ "paramDescription": "when true will stop SparkSession after job execution",
+ "paramRequired": false
+ },
+ {
+ "paramName": "hmu",
+ "paramLongName": "hiveMetastoreUris",
+ "paramDescription": "the hive metastore uris",
+ "paramRequired": true
+ },
+ {
+ "paramName": "sql",
+ "paramLongName": "sql",
+ "paramDescription": "sql script to execute",
+ "paramRequired": true
+ }
+]
\ No newline at end of file
diff --git a/dhp-common/src/main/resources/eu/dnetlib/dhp/schema/oaf/utils/pid_blacklist.json b/dhp-common/src/main/resources/eu/dnetlib/dhp/schema/oaf/utils/pid_blacklist.json
new file mode 100644
index 000000000..05e8cde72
--- /dev/null
+++ b/dhp-common/src/main/resources/eu/dnetlib/dhp/schema/oaf/utils/pid_blacklist.json
@@ -0,0 +1,5 @@
+{
+ "doi" : [ "10.12739/10.12739", "10.11646/zootaxa.4404.1.1", "10.5281/zenodo.3678492", "10.11646/zootaxa.4757.1.1", "10.17176/20170811-142447", "10.6035/asparkia", "10.11646/zootaxa.4754.1.6", "10.11646/zootaxa.4784.1.1", "10.6035/millars", "10.11646/zootaxa.4776.1.1", "10.1590/1982-0224-20170094", "10.11646/zootaxa.4773.1.1", "10.11646/zootaxa.4744.1.1", "10.3897/zookeys.38.383", "10.1371/journal.", "10.5281/zenodo.3727017", "10.5252/zoosystema2019v41a15", "10.6035/dossiersf", "10.11646/zootaxa.4754.1.20", "10.6035/recerca", "10.11646/zootaxa.4428.1.1", "10.7179/psri", "10.11646/zootaxa.4785.1.1", "10.2478/aemnp-2018-0014", "10.17979/spudc.9788497497565", "10.2139/ssrn.2721313", "10.17979/spudc.9788497497749", "10.5281/zenodo.3760976", "10.11646/zootaxa.4381.1.1", "10.6035/tiempos", "10.11646/zootaxa.4754.1.10", "10.5281/zenodo.3776452", "10.11646/zootaxa.4754.1.16", "10.5252/zoosystema2019v41a26", "10.11646/zootaxa.4759.2.1", "10.11646/zootaxa.4741.1.1", "10.5252/zoosystema2019v41a4", "10.1145/nnnnnnn.nnnnnnn", "10.17979/spudc.9788497497169", "10.11646/zootaxa.4780.3.1", "10.11646/zootaxa.4663.1.1", "10.5281/zenodo.3748525", "10.5281/zenodo.3746744", "10.3920/978-90-8686-761-5", "10.14198/eurau18alicante", "10.5252/geodiversitas2019v41a8", "10.4126/38m-0000003", "10.5281/zenodo.3648511", "10.6035/clr", "10.4126/38m-0000004", "10.5281/zenodo.3732535", "10.5281/zenodo.3355776", "10.4126/38m-0000002", "10.11646/zootaxa.4763.3.3", "10.11646/zootaxa.4413.3.1", "10.1163/9789004416208_005", "10.4126/38m-0000001", "10.3897/zookeys.30.308", "10.4126/38m-0000000", "10.5281/zenodo.3739808", "10.5281/zenodo.3674873", "10.3161/00034541anz2020.70.1.003", "10.5281/zenodo.3738648", "10.11646/zootaxa.4765.1.1", "10.11646/zootaxa.4754.1.8", "10.3897/zookeys.36.306", "10.4230/lipics", "10.5281/zenodo.3758345", "10.3161/00034541anz2020.70.1.001", "10.3929/ethz-a-005427569", "10.11646/zootaxa.4772.1.1", "10.5281/zenodo.3677235", "10.11646/zootaxa.4766.1.1", "10.17509/jurnal", "10.1145/1235", "10.11646/zootaxa.4754.1.15", "10.2478/aemnp-2018-0018", "10.11646/zootaxa.4538.1.1", "10.11646/zootaxa.4740.1.1", "10.3897/zookeys.32.282", "10.3897/zookeys.2.56", "10.3897/zookeys.39.425", "10.11646/zootaxa.4514.3.3", "10.1007/978-94-007-1966-8", "10.3897/zookeys.26.214", "10.11646/zootaxa.4106.1.1", "10.3897/zookeys.22.219", "10.11646/zootaxa.4748.2.1", "10.5252/zoosystema2019v41a19", "10.3897/zookeys.22.122", "10.1080/00222933.2019.1634225", "10.11646/zootaxa.4632.1.1", "10.1007/s00259-016-3484-4", "10.3897/zookeys.19.221", "10.3897/zookeys.2.7", "10.11646/zootaxa.4777.1.1", "10.14279/depositonce-3753", "10.1111/apha.12712", "10.11646/zootaxa.4759.3.4", "10.11646/zootaxa.4754.1.9", "10.11646/zootaxa.4747.2.8", "10.5281/zenodo.3757451", "10.5281/zenodo.3740269", "10.5252/zoosystema2020v42a4", "10.1140/epje/i2013-13103-3", "10.1177/0301006619863862", "10.5281/zenodo.3726987", "10.12795/hid", "10.24042/jipf", "10.12795/e-rips", "10.1186/s12913-016-1423-5", "10.4126/38m-0000005", "10.3847/2041-8213/aa91c9", "10.1145/1122445.1122456", "10.1103/physrevlett.114.191803", "10.3920/978-90-8686-782-0", "10.11646/zootaxa.4739.1.1", "10.11646/zootaxa.4770.1.1", "10.21009/10.21009/jpd.081", "10.1080/15548627.2015.1100356", "10.12795/ricl", "10.3897/zookeys.34.309", "10.1080/00222933.2019.1692088", "10.4126/frl01-0064002", "10.1371/journal", "10.1175/1520-0485(2002)032", "10.3897/zookeys.22.152", "10.11646/zootaxa.4731.2.1", "10.4126/frl01-0064005", "10.11646/zootaxa.4738.1.1", "10.11646/zootaxa.4780.1.6", "10.4126/frl01-0064004", "10.6018/analesps.31.1.158071", "10.1007/jhep08(2016)045", "10.5281/zenodo.3759519", "10.4126/frl01-0064010", "10.11646/zootaxa.4537.1.1", "10.5281/zenodo.3713533", "10.5281/zenodo.3742020", "10.4126/frl01-0064014", "10.4126/frl01-0064001", "10.1000/isbn", "10.5281/zenodo.3777290", "10.4126/frl01-0064008", "10.1159/000440895", "10.3897/zookeys.31.140", "10.4126/frl01-0064003", "10.1080/00222933.2018.1524032", "10.21686/2500-3925-2014-6", "10.1016/j.bbr.2011.03.031", "10.4126/frl01-0064006", "10.4126/frl01-0064007", "10.4126/frl01-0064020", "10.4126/frl01-0064016", "10.2478/aemnp-2018-0013", "10.4126/frl01-0064021", "10.5281/zenodo.3754300", "10.15330/gal.29-30.", "10.3897/zookeys.2.4", "10.5252/zoosystema2019v41a7", "10.22435/bpk.v17i2", "10.4126/frl01-0063997", "10.3897/zookeys.11.160", "10.11646/zootaxa.4754.1.14", "10.4126/frl01-0064013", "10.1080/20013078.2018.1535750", "10.1016/j.", "10.4126/frl01-0064011", "10.1002/ece3.2579", "10.1088/0264-9381/28/9/094001", "10.3897/zookeys.2.25", "10.4126/frl01-0064019", "10.4126/frl01-0063994", "10.4126/frl01-0064135", "10.4126/frl01-0063998", "10.12795/ppa", "10.4126/frl01-0064009", "10.11646/zootaxa.4769.1.1", "10.11646/zootaxa.4419.1.1", "10.11646/zootaxa.4733.1.1", "10.4126/frl01-0063993", "10.3161/15081109acc2016.18.1.005", "10.11646/zootaxa.4763.1.2", "10.11646/zootaxa.4754.1.19", "10.4126/frl01-0064136", "10.4126/frl01-0064159", "10.4126/frl01-0063999", "10.4126/frl01-0064161", "10.1089/ten.tea.2015.5000.abstracts", "10.1002/(issn)1521-3773", "10.1140/epjc/s10052-015-3325-9", "10.1016/j.physletb.2016.04.050", "10.1007/jhep04(2015)117", "10.1111/gcb.14904", "10.1016/s0140-6736(17)32129-3", "10.11646/zootaxa.4748.1.1", "10.4126/frl01-0064078", "10.1140/epjc/s10052-015-3408-7", "10.1002/(issn)1097-4652", "10.1007/jhep06(2015)121", "10.1007/jhep09(2014)103", "10.1016/j.gca.2007.06.021", "10.1007/jhep09(2015)049", "10.3897/zookeys.4.32", "10.6101/azq/0002", "10.11646/zootaxa.4764.1.1", "10.11646/zootaxa.4772.1.5", "10.4126/frl01-0064000", "10.4126/frl01-0064131", "10.1016/j.physletb.2015.08.061", "10.1007/jhep01(2015)069", "10.1016/j.physletb.2016.06.039", "10.1016/j.physletb.2015.07.011", "10.1007/jhep04(2015)116", "10.3920/978-90-8686-797-4", "10.1016/j.physletb.2015.12.020", "10.1016/j.physletb.2015.04.042", "10.1016/j.physletb.2016.06.004", "10.1140/epjc/s10052-015-3261-8", "10.1016/j.physletb.2015.10.067", "10.1016/j.physletb.2015.07.065", "10.1163/1876312x-00002195", "10.1016/j.physletb.2013.12.010", "10.1016/j.physletb.2013.01.024", "10.1007/jhep11(2014)056", "10.1007/jhep12(2017)142", "10.1002/pds.4864", "10.1140/epjc/s10052-015-3262-7", "10.1016/j.physletb.2014.09.054", "10.1140/epjc/s10052-015-3373-1", "10.1007/jhep03(2015)041", "10.1016/j.physletb.2016.02.047", "10.4126/frl01-0064018", "10.1016/j.physletb.2014.01.042", "10.1007/jhep09(2014)037", "10.1007/978-94-017-7285-3", "10.1007/s00424-013-1401-2", "10.1007/s00259-017-3822-1", "10.1177/0301006616671273", "10.1007/jhep09(2014)112", "10.1007/jhep06(2015)116", "10.1140/epjc/s10052-018-6243-9", "10.1140/epjc/s10052-017-4692-1", "10.1007/jhep10(2015)144", "10.1007/jhep07(2017)107", "10.1007/jhep11(2014)088", "10.1016/j.physletb.2014.01.006", "10.1007/jhep01(2018)055", "10.1016/j.physletb.2016.03.060", "10.1140/epjc/s10052-019-6904-3", "10.11646/zootaxa.4737.1.1", "10.3934/xx.xx.xx.xx", "10.11646/zootaxa.4758.2.1", "10.1016/j.physletb.2015.10.004", "10.1016/j.physletb.2015.07.053", "10.5798/diclemedj.0921.2012.04.0184", "10.1007/jhep04(2014)169", "10.4126/frl01-0064160", "10.3989/aem.2001.v31.i2", "10.1039/x0xx00000x", "10.11646/zootaxa.3856.4.1", "10.4126/frl01-0064133", "10.1007/jhep05(2015)078", "10.1016/j.physletb.2012.08.020", "10.1007/jhep07(2015)032", "10.1159/000090218", "10.1016/j.physletb.2014.03.015", "10.1007/jhep09(2015)108", "10.1007/jhep09(2015)050", "10.1007/jhep01(2014)163", "10.1016/j.physletb.2014.11.026", "10.1140/epjc/s10052-016-4580-0", "10.1140/epjc/s10052-014-3109-7", "10.1140/epjc/s10052-014-3231-6", "10.1007/jhep02(2014)088", "10.1016/j.physletb.2016.01.056", "10.1016/j.physletb.2015.08.047", "10.1016/j.physletb.2015.12.039", "10.1007/jhep11(2015)071", "10.1140/epjc/s10052-015-3853-3", "10.1007/jhep04(2015)124", "10.1016/j.physletb.2015.07.010", "10.5281/zenodo.3413524", "10.1007/jhep04(2014)031", "10.1007/jhep07(2015)157", "10.1103/physrevd.90.052008", "10.1007/jhep11(2014)118", "10.3920/978-90-8686-708-0", "10.5281/zenodo.1136235", "10.1103/physrevd.86.032003", "10.1016/j.physletb.2016.01.032", "10.1007/jhep03(2018)174", "10.1007/jhep10(2017)182", "10.1140/epjst/e2019-900045-4", "10.1016/j.physletb.2015.06.070", "10.1140/epjc/s10052-016-4067-z", "10.1016/j.physletb.2015.11.042", "10.1007/jhep04(2018)033", "10.1007/jhep09(2014)145", "10.1016/j.physletb.2016.08.055", "10.1016/j.physletb.2015.04.002", "10.1007/jhep03(2014)032", "10.1140/epjc/s10052-017-5491-4", "10.1016/j.physletb.2015.09.062", "10.1016/j.physletb.2014.12.003", "10.1016/j.physletb.2015.03.017", "10.1140/epjc/s10052-014-3195-6", "10.1140/epjc/s10052-016-4034-8", "10.1140/epjc/s10052-016-4070-4", "10.1140/epjc/s10052-018-5693-4", "10.4126/frl01-0064017", "10.1007/jhep08(2014)173", "10.1016/j.physletb.2014.06.076", "10.1016/j.physletb.2018.11.064", "10.1140/epjc/s10052-017-4988-1", "10.11646/zootaxa.4258.4.3", "10.11646/zootaxa.4766.1.2", "10.11646/zootaxa.4780.1.1", "10.5281/zenodo.3693943", "10.4126/frl01-0064129", "10.15330/gal.28.", "10.1007/jhep02(2016)145", "10.1007/jhep04(2014)172", "10.1007/jhep04(2016)005", "10.1007/jhep03(2016)125", "10.1016/j.physletb.2018.02.033", "10.1007/jhep08(2017)052", "10.1007/jhep12(2017)085", "10.1007/jhep09(2014)176", "10.1007/jhep12(2017)024", "10.1140/epjc/s10052-018-5686-3", "10.1016/j.physletb.2016.11.035", "10.1016/j.physletb.2015.12.017", "10.1140/epjc/s10052-015-3542-2", "10.1140/epjc/s10052-014-3071-4", "10.1103/physrevd.97.032009", "10.1140/epjc/s10052-015-3306-z", "10.1016/j.physletb.2017.12.043", "10.1140/epjc/s10052-014-3233-4", "10.1016/j.physletb.2018.09.013", "10.1016/j.gca.2007.06.014", "10.1016/j.physletb.2016.05.005", "10.1038/s41586-019-1171-x", "10.1016/j.physletb.2016.05.087", "10.1007/jhep06(2018)022", "10.1016/j.physletb.2016.01.057", "10.1016/j.physletb.2018.03.023", "10.1140/epjc/s10052-015-3351-7", "10.1126/science.aap8757", "10.1007/jhep09(2015)137", "10.1007/jhep01(2015)063", "10.1007/jhep01(2018)126", "10.1016/j.gca.2007.06.020", "10.1140/epjc/s10052-018-5595-5", "10.1016/j.physletb.2015.02.015", "10.1016/j.physletb.2014.06.077", "10.1007/jhep12(2017)059", "10.1007/jhep10(2017)141", "10.1007/jhep02(2014)107", "10.1140/epjc/s10052-014-2965-5", "10.1016/j.physletb.2015.07.079", "10.1007/jhep10(2017)112", "10.1140/epjc/s10052-014-2982-4", "10.1007/jhep05(2016)160", "10.1016/j.physletb.2016.07.030", "10.1140/epjc/s10052-014-3168-9", "10.1140/epjc/s10052-018-5583-9", "10.1140/epjc/s10052-016-4184-8", "10.1007/jhep08(2015)105", "10.1007/jhep05(2015)061", "10.1103/physrevd.97.032003", "10.1140/epjc/s10052-014-3190-y", "10.1016/j.physletb.2012.10.061", "10.1140/epjc/s10052-014-2941-0", "10.1016/j.physletb.2016.02.002", "10.1016/j.physletb.2016.05.033", "10.1007/jhep01(2014)096", "10.1007/jhep09(2015)201", "10.1016/j.physletb.2016.01.010", "10.1016/j.physletb.2015.07.037", "10.1007/jhep07(2015)042", "10.1016/j.physletb.2016.05.044", "10.1016/j.physletb.2016.05.088", "10.3897/zookeys.2.2", "10.1007/jhep11(2015)018", "10.1007/jhep11(2015)189", "10.1016/j.physletb.2016.10.014", "10.1007/jhep06(2015)080", "10.1016/j.physletb.2014.11.042", "10.1140/epjc/s10052-014-3157-z", "10.1140/epjc/s10052-015-3406-9", "10.1016/j.physletb.2016.02.056", "10.1016/j.physletb.2015.03.054", "10.1140/epjc/s10052-016-4574-y", "10.5252/geodiversitas2019v41a15", "10.1007/jhep09(2014)094", "10.1140/epjc/s10052-017-5486-1", "10.1007/jhep03(2018)095", "10.11646/zootaxa.4736.1.1", "10.11646/zootaxa.4766.2.1", "10.5281/zenodo.3762392", "10.5281/zenodo.3761958", "10.11646/zootaxa.4403.3.2", "10.1553/iswimab", "10.11646/zootaxa.3750.5.1", "10.4126/frl01-0064134", "10.1103/physrevd.87.032002", "10.1140/epjc/s10052-013-2676-3", "10.1007/jhep02(2015)153", "10.1007/jhep08(2017)006", "10.1016/j.physletb.2016.11.005", "10.1007/jhep01(2013)029", "10.1007/jhep10(2017)132", "10.1016/j.physletb.2013.01.034", "10.1016/j.physletb.2016.03.046", "10.1140/epjc/s10052-016-3988-x", "10.1016/j.physletb.2016.07.006", "10.1140/epjc/s10052-018-5752-x", "10.1140/epjc/s10052-015-3454-1", "10.1002/ece3.1303", "10.1007/jhep02(2014)013", "10.1007/jhep06(2016)081", "10.1140/epjc/s10052-014-3117-7", "10.1007/jhep09(2017)084", "10.1016/j.physletb.2017.09.078", "10.1007/jhep08(2016)005", "10.1007/jhep01(2015)020", "10.1140/epjc/s10052-017-4852-3", "10.1016/j.physletb.2018.02.045", "10.7818/sibecolandaeetmeeting.2019", "10.1007/jhep11(2014)104", "10.1007/jhep05(2018)077", "10.1016/j.physletb.2016.11.045", "10.1016/j.physletb.2016.10.042", "10.1140/epjc/s10052-016-4203-9", "10.1007/jhep01(2015)068", "10.1007/jhep06(2016)093", "10.1016/j.physletb.2015.09.051", "10.1140/epjc/s10052-015-3534-2", "10.1007/jhep09(2014)087", "10.1016/j.physletb.2014.05.055", "10.1016/j.physletb.2014.02.033", "10.1140/epjc/s10052-017-5225-7", "10.1140/epjc/s10052-017-5442-0", "10.1016/s0140-6736(18)32335-3", "10.1016/j.physletb.2017.11.049", "10.1007/jhep06(2018)166", "10.1016/j.physletb.2016.05.002", "10.1140/epjc/s10052-016-4219-1", "10.1140/epjst/e2019-900087-0", "10.1007/jhep01(2016)166", "10.1007/jhep01(2018)097", "10.1016/j.physletb.2017.11.043", "10.1016/j.physletb.2018.04.036", "10.1140/epjc/s10052-018-5607-5", "10.1007/jhep12(2017)034", "10.1007/jhep11(2016)112", "10.1007/jhep06(2014)008", "10.1140/epjc/s10052-012-2261-1", "10.1016/j.physletb.2014.08.039", "10.1016/s0140-6736(16)31919-5", "10.1140/epjc/s10052-019-7058-z", "10.1016/j.physletb.2014.07.053", "10.1007/jhep01(2015)053", "10.1016/j.physletb.2016.07.042", "10.1007/jhep08(2014)103", "10.1007/jhep06(2015)100", "10.1140/epjc/s10052-015-3363-3", "10.1140/epjc/s10052-017-4915-5", "10.1140/epjc/s10052-014-3023-z", "10.1140/epjc/s10052-017-5315-6", "10.1140/epjc/s10052-016-4050-8", "10.3389/fpsyt.2017.00244", "10.1016/j.physletb.2014.10.002", "10.1007/jhep07(2015)162", "10.1007/jhep08(2014)174", "10.3897/zookeys.2.23", "10.1007/jhep07(2017)014", "10.1007/jhep04(2016)035", "10.1140/epjc/s10052-017-4984-5", "10.1007/jhep02(2016)156", "10.1016/j.physletb.2016.03.039", "10.1007/jhep07(2018)115", "10.3897/zookeys.34.268", "10.1007/jhep02(2016)122", "10.1016/j.physletb.2012.03.022", "10.1016/j.physletb.2018.09.019", "10.1016/j.physletb.2018.09.024", "10.1051/0004-6361/201629272", "10.1103/physrevc.97.024904", "10.1140/epjc/s10052-016-4521-y", "10.1140/epjc/s10052-016-4176-8", "10.1140/epjc/s10052-014-3134-6", "10.1140/epjc/s10052-016-4110-0", "10.1007/jhep07(2017)121", "10.1007/jhep07(2018)153", "10.1007/jhep03(2018)115", "10.1007/jhep04(2018)060", "10.11606/1807-0205/2020.60.06", "10.4126/frl01-0064015", "10.1007/jhep09(2017)020", "10.1016/j.physletb.2014.04.023", "10.1016/j.physletb.2015.02.048", "10.1007/jhep02(2018)032", "10.1016/j.physletb.2018.01.001", "10.1140/epjc/s10052-015-3852-4", "10.1007/jhep10(2014)087", "10.11646/zootaxa.4630.1.1", "10.5281/zenodo.3742118", "10.4126/frl01-0064022", "10.11646/zootaxa.4758.3.1", "10.11646/zootaxa.4772.3.1", "10.11646/zootaxa.4576.3.5", "10.4126/frl01-0064125", "10.1007/jhep12(2017)017", "10.4126/frl01-0064162", "10.4126/frl01-0064138", "10.1007/jhep06(2014)124", "10.1007/jhep06(2016)059", "10.1007/jhep06(2014)035", "10.1103/physrevd.90.052005", "10.1007/jhep11(2017)062", "10.3847/2041-8213/aa9aed", "10.1016/j.physletb.2016.06.080", "10.1007/jhep10(2017)073", "10.1007/jhep03(2018)167", "10.1016/j.physletb.2018.11.065", "10.1140/epjc/s10052-017-5081-5", "10.1140/epjc/s10052-015-3500-z", "10.1140/epjc/s10052-017-5445-x", "10.1016/j.physletb.2014.01.049", "10.1007/jhep03(2018)172", "10.1016/j.physletb.2015.03.048", "10.1016/j.physletb.2018.11.032", "10.1007/jhep05(2018)025", "10.1016/j.physletb.2016.08.052", "10.1016/j.physletb.2014.09.008", "10.1103/physrevlett.120.071802", "10.1016/j.physletb.2018.01.049", "10.1016/j.physletb.2016.06.017", "10.1016/j.physletb.2016.04.005", "10.1007/jhep06(2018)031", "10.1007/jhep01(2016)079", "10.1007/jhep10(2017)006", "10.1140/epjc/s10052-018-5740-1", "10.1016/j.physletb.2015.01.034", "10.1007/jhep10(2017)005", "10.1016/j.physletb.2018.04.007", "10.1007/jhep04(2015)164", "10.1140/epjc/s10052-018-5691-6", "10.1007/jhep05(2018)148", "10.1007/jhep03(2018)003", "10.1140/epjc/s10052-014-3076-z", "10.1016/j.physletb.2016.02.015", "10.1103/physrevd.97.072003", "10.1016/j.physletb.2017.11.054", "10.1140/epjc/s10052-011-1849-1", "10.1007/jhep09(2016)175", "10.1016/j.physletb.2017.12.011", "10.1007/jhep04(2014)103", "10.1007/jhep12(2014)017", "10.1016/j.physletb.2014.09.048", "10.1140/epjc/s10052-019-7202-9", "10.1007/jhep04(2014)191", "10.1007/jhep07(2013)163", "10.1140/epjc/s10052-014-3130-x", "10.1007/jhep04(2016)023", "10.1016/j.physletb.2015.07.023", "10.1140/epjc/s10052-018-6500-y", "10.1016/j.physletb.2015.04.045", "10.1007/jhep09(2017)053", "10.1007/jhep10(2017)180", "10.1140/epjc/s10052-017-4912-8", "10.1007/jhep10(2016)129", "10.3920/978-90-8686-816-2", "10.1007/jhep01(2017)099", "10.1007/jhep01(2018)045", "10.1007/jhep04(2015)025", "10.1016/j.physletb.2018.02.050", "10.1103/physrevlett.116.032301", "10.1007/jhep08(2017)029", "10.1007/jhep08(2017)073", "10.1016/j.physletb.2014.11.059", "10.1007/jhep01(2013)131", "10.1007/jhep06(2014)112", "10.1016/j.physletb.2017.09.066", "10.1140/epjc/s10052-014-2883-6", "10.1094/mpmi", "10.1007/jhep11(2017)195", "10.1007/jhep06(2018)108", "10.1007/jhep09(2018)139", "10.1016/j.physletb.2016.12.005", "10.1140/epjc/s10052-017-5349-9", "10.1016/j.physletb.2012.08.021", "10.1016/j.physletb.2014.10.032", "10.1007/jhep09(2017)088", "10.1140/epjc/s10052-015-3425-6", "10.1007/jhep01(2018)054", "10.1103/physrevlett.110.182302", "10.1140/epjc/s10052-017-5317-4", "10.1007/jhep01(2017)117", "10.1016/j.physletb.2017.12.006", "10.1016/j.physletb.2018.02.004", "10.1016/j.physletb.2018.02.025", "10.1016/j.physletb.2016.02.055", "10.1016/j.physletb.2016.04.061", "10.1140/epjc/s10052-015-3372-2", "10.1016/j.physletb.2015.02.051", "10.1016/j.physletb.2014.11.049", "10.1007/jhep09(2016)001", "10.1016/j.physletb.2016.03.017", "10.1007/jhep06(2016)067", "10.1140/epjc/s10052-015-3543-1", "10.1140/epjc/s10052-017-4911-9", "10.1007/jhep07(2013)122", "10.1140/epjc/s10052-019-6855-8", "10.1140/epjc/s10052-019-6540-y", "10.1007/jhep06(2014)009", "10.1007/jhep05(2019)043", "10.1016/j.physletb.2016.01.028", "10.1103/physrevlett.120.231801", "10.1140/epjc/s10052-016-4325-0", "10.1007/jhep07(2018)127", "10.1016/j.physletb.2016.05.003", "10.1140/epjc/s10052-017-4644-9", "10.1140/epjc/s10052-017-4700-5", "10.1007/jhep06(2018)107", "10.1016/j.physletb.2018.01.042", "10.1140/epjc/s10052-018-5624-4", "10.1007/jhep08(2016)139", "10.1007/jhep05(2018)195", "10.1103/physrevd.97.052012", "10.1140/epjc/s10052-016-3978-z", "10.1007/jhep05(2019)088", "10.1140/epjc/s10052-017-5079-z", "10.1140/epjc/s10052-016-4205-7", "10.1007/jhep01(2016)006", "10.1140/epjc/s10052-016-4286-3", "10.1016/j.physletb.2017.04.071", "10.1103/physrevd.97.012007", "10.1016/j.physletb.2018.01.077", "10.1007/jhep04(2018)073", "10.1016/j.physletb.2015.09.057", "10.1007/jhep07(2018)032", "10.1140/epjc/s10052-015-3435-4", "10.1007/jhep11(2017)010", "10.1093/isd/ixaa002", "10.1016/j.physletb.2018.03.035", "10.1007/jhep10(2018)031", "10.1016/s0140-6736(18)31891-9", "10.1140/epjc/s10052-018-6148-7", "10.1016/j.physletb.2018.03.057", "10.1140/epjc/s10052-019-6632-8", "10.1016/j.physletb.2015.11.071", "10.1140/epjc/s10052-018-5605-7", "10.1016/j.physletb.2018.10.073", "10.1140/epjc/s10052-019-7387-y", "10.1007/jhep06(2019)143", "10.1140/epjc/s10052-018-5567-9", "10.1140/epjc/s10052-019-6909-y", "10.1002/(sici)1521-3978(199901)47:1/3", "10.5281/zenodo.3758372", "10.4126/frl01-0064041", "10.1140/epjc/s10052-014-3129-3", "10.11646/zootaxa.4685.1.1", "10.11646/zootaxa.4756.1.1", "10.6101/azq/0001", "10.14582/duzg", "10.1016/j.physletb.2012.11.039", "10.4126/frl01-0064191", "10.1016/j.physletb.2013.12.029", "10.1007/jhep10(2013)189", "10.1051/0004-6361/201629512", "10.1007/jhep01(2013)116", "10.2312/gfz.lis.2016.001", "10.1016/j.physletb.2013.01.040", "10.1103/physrevd.90.112005", "10.1140/epjc/s10052-015-3726-9", "10.1007/s11682-013-9269-5", "10.1007/jhep02(2017)071", "10.1016/j.physletb.2016.09.040", "10.1007/jhep02(2017)117", "10.1007/jhep08(2016)009", "10.1103/physrevd.97.052010", "10.1007/jhep09(2017)032", "10.1103/physrevd.97.032005", "10.1140/epjc/s10052-017-4965-8", "10.1016/j.physletb.2016.08.042", "10.1016/j.physletb.2017.10.039", "10.1007/jhep03(2016)127", "10.1140/epjc/s10052-014-3034-9", "10.1007/jhep03(2017)113", "10.1007/jhep11(2018)040", "10.1140/epjc/s10052-018-6457-x", "10.1140/epjc/s10052-016-4041-9", "10.1140/epjc/s10052-018-6219-9", "10.1140/epjc/s10052-016-4149-y", "10.1007/jhep10(2017)072", "10.1140/epjc/s10052-016-4083-z", "10.1140/epjc/s10052-016-3956-5", "10.1007/jhep04(2016)073", "10.1007/jhep06(2016)177", "10.1016/j.physletb.2018.03.084", "10.1007/jhep10(2015)128", "10.1007/jhep03(2018)166", "10.1140/epjc/s10052-015-3491-9", "10.1016/j.physletb.2015.04.060", "10.1103/physrevd.92.112001", "10.1140/epjc/s10052-015-3367-z", "10.1007/jhep10(2017)019", "10.1007/jhep10(2017)131", "10.1016/j.physletb.2018.08.057", "10.1007/jhep01(2016)096", "10.1016/j.physletb.2017.09.053", "10.1007/jhep07(2017)013", "10.1007/jhep01(2019)030", "10.1007/jhep11(2016)110", "10.1016/j.physletb.2012.02.044", "10.1140/epjc/s10052-017-5192-z", "10.1007/jhep03(2015)022", "10.1140/epjc/s10052-019-6847-8", "10.1093/database/baz085", "10.1140/epjc/s10052-015-3451-4", "10.1007/jhep11(2017)029", "10.1140/epjc/s10052-015-3517-3", "10.1007/jhep07(2017)001", "10.1007/jhep09(2016)074", "10.1103/physrevd.97.072016", "10.1007/jhep05(2018)006", "10.1103/physrevlett.120.081801", "10.1103/physrevlett.120.161802", "10.1103/physrevlett.120.202005", "10.5281/zenodo.1299123", "10.5281/zenodo.3777294" ],
+ "pmid" : [],
+ "pmc" : []
+}
\ No newline at end of file
diff --git a/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala b/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala
new file mode 100644
index 000000000..f8afe9af4
--- /dev/null
+++ b/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala
@@ -0,0 +1,73 @@
+package eu.dnetlib.dhp.application
+
+import scala.io.Source
+
+/** This is the main Interface SparkApplication
+ * where all the Spark Scala class should inherit
+ */
+trait SparkScalaApplication {
+
+ /** This is the path in the classpath of the json
+ * describes all the argument needed to run
+ */
+ val propertyPath: String
+
+ /** Utility to parse the arguments using the
+ * property json in the classpath identified from
+ * the variable propertyPath
+ *
+ * @param args the list of arguments
+ */
+ def parseArguments(args: Array[String]): ArgumentApplicationParser = {
+ val parser = new ArgumentApplicationParser(
+ Source.fromInputStream(getClass.getResourceAsStream(propertyPath)).mkString
+ )
+ parser.parseArgument(args)
+ parser
+ }
+
+ /** Here all the spark applications runs this method
+ * where the whole logic of the spark node is defined
+ */
+ def run(): Unit
+}
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.SparkSession
+import org.slf4j.Logger
+
+abstract class AbstractScalaApplication(
+ val propertyPath: String,
+ val args: Array[String],
+ log: Logger
+) extends SparkScalaApplication {
+
+ var parser: ArgumentApplicationParser = null
+
+ var spark: SparkSession = null
+
+ def initialize(): SparkScalaApplication = {
+ parser = parseArguments(args)
+ spark = createSparkSession()
+ this
+ }
+
+ /** Utility for creating a spark session starting from parser
+ *
+ * @return a spark Session
+ */
+ private def createSparkSession(): SparkSession = {
+ require(parser != null)
+
+ val conf: SparkConf = new SparkConf()
+ val master = parser.get("master")
+ log.info(s"Creating Spark session: Master: $master")
+ SparkSession
+ .builder()
+ .config(conf)
+ .appName(getClass.getSimpleName)
+ .master(master)
+ .getOrCreate()
+ }
+
+}
diff --git a/dhp-common/src/main/scala/eu/dnetlib/dhp/application/dedup/log/DedupLogModel.scala b/dhp-common/src/main/scala/eu/dnetlib/dhp/application/dedup/log/DedupLogModel.scala
new file mode 100644
index 000000000..d74ec3f69
--- /dev/null
+++ b/dhp-common/src/main/scala/eu/dnetlib/dhp/application/dedup/log/DedupLogModel.scala
@@ -0,0 +1,10 @@
+package eu.dnetlib.dhp.application.dedup.log
+
+case class DedupLogModel(
+ tag: String,
+ configuration: String,
+ entity: String,
+ startTS: Long,
+ endTS: Long,
+ totalMs: Long
+) {}
diff --git a/dhp-common/src/main/scala/eu/dnetlib/dhp/application/dedup/log/DedupLogWriter.scala b/dhp-common/src/main/scala/eu/dnetlib/dhp/application/dedup/log/DedupLogWriter.scala
new file mode 100644
index 000000000..4409c01d9
--- /dev/null
+++ b/dhp-common/src/main/scala/eu/dnetlib/dhp/application/dedup/log/DedupLogWriter.scala
@@ -0,0 +1,14 @@
+package eu.dnetlib.dhp.application.dedup.log
+
+import org.apache.spark.sql.{SaveMode, SparkSession}
+
+class DedupLogWriter(path: String) {
+
+ def appendLog(dedupLogModel: DedupLogModel, spark: SparkSession): Unit = {
+ import spark.implicits._
+ val df = spark.createDataset[DedupLogModel](data = List(dedupLogModel))
+ df.write.mode(SaveMode.Append).save(path)
+
+ }
+
+}
diff --git a/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala b/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala
new file mode 100644
index 000000000..a995016a8
--- /dev/null
+++ b/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala
@@ -0,0 +1,442 @@
+package eu.dnetlib.dhp.sx.graph.scholix
+
+import eu.dnetlib.dhp.schema.oaf.{Publication, Relation, Result, StructuredProperty}
+import eu.dnetlib.dhp.schema.sx.scholix._
+import eu.dnetlib.dhp.schema.sx.summary.{CollectedFromType, SchemeValue, ScholixSummary, Typology}
+import eu.dnetlib.dhp.utils.DHPUtils
+import org.apache.spark.sql.expressions.Aggregator
+import org.apache.spark.sql.{Encoder, Encoders}
+import org.json4s
+import org.json4s.DefaultFormats
+import org.json4s.jackson.JsonMethods.parse
+import scala.collection.JavaConverters._
+import scala.io.Source
+
+object ScholixUtils extends Serializable {
+
+ val DNET_IDENTIFIER_SCHEMA: String = "DNET Identifier"
+
+ val DATE_RELATION_KEY: String = "RelationDate"
+
+ case class RelationVocabulary(original: String, inverse: String) {}
+
+ case class RelatedEntities(id: String, relatedDataset: Long, relatedPublication: Long) {}
+
+ val relations: Map[String, RelationVocabulary] = {
+ val input = Source
+ .fromInputStream(
+ getClass.getResourceAsStream("/eu/dnetlib/scholexplorer/relation/relations.json")
+ )
+ .mkString
+ implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
+
+ lazy val json: json4s.JValue = parse(input)
+
+ json.extract[Map[String, RelationVocabulary]]
+ }
+
+ def extractRelationDate(relation: Relation): String = {
+
+ if (relation.getProperties == null || !relation.getProperties.isEmpty)
+ null
+ else {
+ val date = relation.getProperties.asScala
+ .find(p => DATE_RELATION_KEY.equalsIgnoreCase(p.getKey))
+ .map(p => p.getValue)
+ if (date.isDefined)
+ date.get
+ else
+ null
+ }
+ }
+
+ def extractRelationDate(summary: ScholixSummary): String = {
+
+ if (summary.getDate == null || summary.getDate.isEmpty)
+ null
+ else {
+ summary.getDate.get(0)
+ }
+ }
+
+ def inverseRelationShip(rel: ScholixRelationship): ScholixRelationship = {
+ new ScholixRelationship(rel.getInverse, rel.getSchema, rel.getName)
+
+ }
+
+ def generateScholixResourceFromResult(r: Result): ScholixResource = {
+ generateScholixResourceFromSummary(ScholixUtils.resultToSummary(r))
+ }
+
+ val statsAggregator: Aggregator[(String, String, Long), RelatedEntities, RelatedEntities] =
+ new Aggregator[(String, String, Long), RelatedEntities, RelatedEntities] with Serializable {
+ override def zero: RelatedEntities = null
+
+ override def reduce(b: RelatedEntities, a: (String, String, Long)): RelatedEntities = {
+ val relatedDataset = if ("dataset".equalsIgnoreCase(a._2)) a._3 else 0
+ val relatedPublication = if ("publication".equalsIgnoreCase(a._2)) a._3 else 0
+
+ if (b == null)
+ RelatedEntities(a._1, relatedDataset, relatedPublication)
+ else
+ RelatedEntities(
+ a._1,
+ b.relatedDataset + relatedDataset,
+ b.relatedPublication + relatedPublication
+ )
+ }
+
+ override def merge(b1: RelatedEntities, b2: RelatedEntities): RelatedEntities = {
+ if (b1 != null && b2 != null)
+ RelatedEntities(
+ b1.id,
+ b1.relatedDataset + b2.relatedDataset,
+ b1.relatedPublication + b2.relatedPublication
+ )
+ else if (b1 != null)
+ b1
+ else
+ b2
+ }
+
+ override def finish(reduction: RelatedEntities): RelatedEntities = reduction
+
+ override def bufferEncoder: Encoder[RelatedEntities] = Encoders.bean(classOf[RelatedEntities])
+
+ override def outputEncoder: Encoder[RelatedEntities] = Encoders.bean(classOf[RelatedEntities])
+ }
+
+ val scholixAggregator: Aggregator[(String, Scholix), Scholix, Scholix] =
+ new Aggregator[(String, Scholix), Scholix, Scholix] with Serializable {
+ override def zero: Scholix = null
+
+ def scholix_complete(s: Scholix): Boolean = {
+ if (s == null || s.getIdentifier == null) {
+ false
+ } else if (s.getSource == null || s.getTarget == null) {
+ false
+ } else if (s.getLinkprovider == null || s.getLinkprovider.isEmpty)
+ false
+ else
+ true
+ }
+
+ override def reduce(b: Scholix, a: (String, Scholix)): Scholix = {
+ if (scholix_complete(b)) b else a._2
+ }
+
+ override def merge(b1: Scholix, b2: Scholix): Scholix = {
+ if (scholix_complete(b1)) b1 else b2
+ }
+
+ override def finish(reduction: Scholix): Scholix = reduction
+
+ override def bufferEncoder: Encoder[Scholix] = Encoders.kryo[Scholix]
+
+ override def outputEncoder: Encoder[Scholix] = Encoders.kryo[Scholix]
+ }
+
+ def createInverseScholixRelation(scholix: Scholix): Scholix = {
+ val s = new Scholix
+ s.setPublicationDate(scholix.getPublicationDate)
+ s.setPublisher(scholix.getPublisher)
+ s.setLinkprovider(scholix.getLinkprovider)
+ s.setRelationship(inverseRelationShip(scholix.getRelationship))
+ s.setSource(scholix.getTarget)
+ s.setTarget(scholix.getSource)
+ s.setIdentifier(
+ DHPUtils.md5(
+ s"${s.getSource.getIdentifier}::${s.getRelationship.getName}::${s.getTarget.getIdentifier}"
+ )
+ )
+ s
+
+ }
+
+ def extractCollectedFrom(summary: ScholixResource): List[ScholixEntityId] = {
+ if (summary.getCollectedFrom != null && !summary.getCollectedFrom.isEmpty) {
+ val l: List[ScholixEntityId] = summary.getCollectedFrom.asScala.map { d =>
+ new ScholixEntityId(d.getProvider.getName, d.getProvider.getIdentifiers)
+ }(collection.breakOut)
+ l
+ } else List()
+ }
+
+ def extractCollectedFrom(summary: ScholixSummary): List[ScholixEntityId] = {
+ if (summary.getDatasources != null && !summary.getDatasources.isEmpty) {
+ val l: List[ScholixEntityId] = summary.getDatasources.asScala.map { d =>
+ new ScholixEntityId(
+ d.getDatasourceName,
+ List(new ScholixIdentifier(d.getDatasourceId, "DNET Identifier", null)).asJava
+ )
+ }(collection.breakOut)
+ l
+ } else List()
+ }
+
+ def extractCollectedFrom(relation: Relation): List[ScholixEntityId] = {
+ if (relation.getCollectedfrom != null && !relation.getCollectedfrom.isEmpty) {
+
+ val l: List[ScholixEntityId] = relation.getCollectedfrom.asScala.map { c =>
+ new ScholixEntityId(
+ c.getValue,
+ List(new ScholixIdentifier(c.getKey, DNET_IDENTIFIER_SCHEMA, null)).asJava
+ )
+ }.toList
+ l
+ } else List()
+ }
+
+ def generateCompleteScholix(scholix: Scholix, target: ScholixSummary): Scholix = {
+ val s = new Scholix
+ s.setPublicationDate(scholix.getPublicationDate)
+ s.setPublisher(scholix.getPublisher)
+ s.setLinkprovider(scholix.getLinkprovider)
+ s.setRelationship(scholix.getRelationship)
+ s.setSource(scholix.getSource)
+ s.setTarget(generateScholixResourceFromSummary(target))
+ s.setIdentifier(
+ DHPUtils.md5(
+ s"${s.getSource.getIdentifier}::${s.getRelationship.getName}::${s.getTarget.getIdentifier}"
+ )
+ )
+ s
+ }
+
+ def generateCompleteScholix(scholix: Scholix, target: ScholixResource): Scholix = {
+ val s = new Scholix
+ s.setPublicationDate(scholix.getPublicationDate)
+ s.setPublisher(scholix.getPublisher)
+ s.setLinkprovider(scholix.getLinkprovider)
+ s.setRelationship(scholix.getRelationship)
+ s.setSource(scholix.getSource)
+ s.setTarget(target)
+ s.setIdentifier(
+ DHPUtils.md5(
+ s"${s.getSource.getIdentifier}::${s.getRelationship.getName}::${s.getTarget.getIdentifier}"
+ )
+ )
+ s
+ }
+
+ def generateScholixResourceFromSummary(summaryObject: ScholixSummary): ScholixResource = {
+ val r = new ScholixResource
+ r.setIdentifier(summaryObject.getLocalIdentifier)
+ r.setDnetIdentifier(summaryObject.getId)
+
+ r.setObjectType(summaryObject.getTypology.toString)
+ r.setObjectSubType(summaryObject.getSubType)
+
+ if (summaryObject.getTitle != null && !summaryObject.getTitle.isEmpty)
+ r.setTitle(summaryObject.getTitle.get(0))
+
+ if (summaryObject.getAuthor != null && !summaryObject.getAuthor.isEmpty) {
+ val l: List[ScholixEntityId] =
+ summaryObject.getAuthor.asScala.map(a => new ScholixEntityId(a, null)).toList
+ if (l.nonEmpty)
+ r.setCreator(l.asJava)
+ }
+
+ if (summaryObject.getDate != null && !summaryObject.getDate.isEmpty)
+ r.setPublicationDate(summaryObject.getDate.get(0))
+ if (summaryObject.getPublisher != null && !summaryObject.getPublisher.isEmpty) {
+ val plist: List[ScholixEntityId] =
+ summaryObject.getPublisher.asScala.map(p => new ScholixEntityId(p, null)).toList
+
+ if (plist.nonEmpty)
+ r.setPublisher(plist.asJava)
+ }
+
+ if (summaryObject.getDatasources != null && !summaryObject.getDatasources.isEmpty) {
+
+ val l: List[ScholixCollectedFrom] = summaryObject.getDatasources.asScala
+ .map(c =>
+ new ScholixCollectedFrom(
+ new ScholixEntityId(
+ c.getDatasourceName,
+ List(new ScholixIdentifier(c.getDatasourceId, DNET_IDENTIFIER_SCHEMA, null)).asJava
+ ),
+ "collected",
+ "complete"
+ )
+ )
+ .toList
+
+ if (l.nonEmpty)
+ r.setCollectedFrom(l.asJava)
+
+ }
+ r
+ }
+
+ def scholixFromSource(relation: Relation, source: ScholixResource): Scholix = {
+ if (relation == null || source == null)
+ return null
+ val s = new Scholix
+ var l: List[ScholixEntityId] = extractCollectedFrom(relation)
+ if (l.isEmpty)
+ l = extractCollectedFrom(source)
+ if (l.isEmpty)
+ return null
+ s.setLinkprovider(l.asJava)
+ var d = extractRelationDate(relation)
+ if (d == null)
+ d = source.getPublicationDate
+
+ s.setPublicationDate(d)
+
+ if (source.getPublisher != null && !source.getPublisher.isEmpty) {
+ s.setPublisher(source.getPublisher)
+ }
+
+ val semanticRelation = relations.getOrElse(relation.getRelClass.toLowerCase, null)
+ if (semanticRelation == null)
+ return null
+ s.setRelationship(
+ new ScholixRelationship(semanticRelation.original, "datacite", semanticRelation.inverse)
+ )
+ s.setSource(source)
+
+ s
+ }
+
+ def scholixFromSource(relation: Relation, source: ScholixSummary): Scholix = {
+
+ if (relation == null || source == null)
+ return null
+
+ val s = new Scholix
+
+ var l: List[ScholixEntityId] = extractCollectedFrom(relation)
+ if (l.isEmpty)
+ l = extractCollectedFrom(source)
+ if (l.isEmpty)
+ return null
+
+ s.setLinkprovider(l.asJava)
+
+ var d = extractRelationDate(relation)
+ if (d == null)
+ d = extractRelationDate(source)
+
+ s.setPublicationDate(d)
+
+ if (source.getPublisher != null && !source.getPublisher.isEmpty) {
+ val l: List[ScholixEntityId] = source.getPublisher.asScala
+ .map { p =>
+ new ScholixEntityId(p, null)
+ }(collection.breakOut)
+
+ if (l.nonEmpty)
+ s.setPublisher(l.asJava)
+ }
+
+ val semanticRelation = relations.getOrElse(relation.getRelClass.toLowerCase, null)
+ if (semanticRelation == null)
+ return null
+ s.setRelationship(
+ new ScholixRelationship(semanticRelation.original, "datacite", semanticRelation.inverse)
+ )
+ s.setSource(generateScholixResourceFromSummary(source))
+
+ s
+ }
+
+ def findURLForPID(
+ pidValue: List[StructuredProperty],
+ urls: List[String]
+ ): List[(StructuredProperty, String)] = {
+ pidValue.map { p =>
+ val pv = p.getValue
+
+ val r = urls.find(u => u.toLowerCase.contains(pv.toLowerCase))
+ (p, r.orNull)
+ }
+ }
+
+ def extractTypedIdentifierFromInstance(r: Result): List[ScholixIdentifier] = {
+ if (r.getInstance() == null || r.getInstance().isEmpty)
+ return List()
+ r.getInstance()
+ .asScala
+ .filter(i => i.getUrl != null && !i.getUrl.isEmpty)
+ .filter(i => i.getPid != null && i.getUrl != null)
+ .flatMap(i => findURLForPID(i.getPid.asScala.toList, i.getUrl.asScala.toList))
+ .map(i => new ScholixIdentifier(i._1.getValue, i._1.getQualifier.getClassid, i._2))
+ .distinct
+ .toList
+ }
+
+ def resultToSummary(r: Result): ScholixSummary = {
+ val s = new ScholixSummary
+ s.setId(r.getId)
+ if (r.getPid == null || r.getPid.isEmpty)
+ return null
+
+ val persistentIdentifiers: List[ScholixIdentifier] = extractTypedIdentifierFromInstance(r)
+ if (persistentIdentifiers.isEmpty)
+ return null
+ s.setLocalIdentifier(persistentIdentifiers.asJava)
+ if (r.isInstanceOf[Publication])
+ s.setTypology(Typology.publication)
+ else
+ s.setTypology(Typology.dataset)
+
+ s.setSubType(r.getInstance().get(0).getInstancetype.getClassname)
+
+ if (r.getTitle != null && r.getTitle.asScala.nonEmpty) {
+ val titles: List[String] = r.getTitle.asScala.map(t => t.getValue).toList
+ if (titles.nonEmpty)
+ s.setTitle(titles.asJava)
+ else
+ return null
+ }
+
+ if (r.getAuthor != null && !r.getAuthor.isEmpty) {
+ val authors: List[String] = r.getAuthor.asScala.map(a => a.getFullname).toList
+ if (authors.nonEmpty)
+ s.setAuthor(authors.asJava)
+ }
+ if (r.getInstance() != null) {
+ val dt: List[String] = r
+ .getInstance()
+ .asScala
+ .filter(i => i.getDateofacceptance != null)
+ .map(i => i.getDateofacceptance.getValue)
+ .toList
+ if (dt.nonEmpty)
+ s.setDate(dt.distinct.asJava)
+ }
+ if (r.getDescription != null && !r.getDescription.isEmpty) {
+ val d = r.getDescription.asScala.find(f => f != null && f.getValue != null)
+ if (d.isDefined)
+ s.setDescription(d.get.getValue)
+ }
+
+ if (r.getSubject != null && !r.getSubject.isEmpty) {
+ val subjects: List[SchemeValue] = r.getSubject.asScala
+ .map(s => new SchemeValue(s.getQualifier.getClassname, s.getValue))
+ .toList
+ if (subjects.nonEmpty)
+ s.setSubject(subjects.asJava)
+ }
+
+ if (r.getPublisher != null)
+ s.setPublisher(List(r.getPublisher.getValue).asJava)
+
+ if (r.getCollectedfrom != null && !r.getCollectedfrom.isEmpty) {
+ val cf: List[CollectedFromType] = r.getCollectedfrom.asScala
+ .map(c => new CollectedFromType(c.getValue, c.getKey, "complete"))
+ .toList
+ if (cf.nonEmpty)
+ s.setDatasources(cf.distinct.asJava)
+ }
+
+ s.setRelatedDatasets(0)
+ s.setRelatedPublications(0)
+ s.setRelatedUnknown(0)
+
+ s
+ }
+
+}
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/application/ArgumentApplicationParserTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/application/ArgumentApplicationParserTest.java
index e14020830..1788239f2 100644
--- a/dhp-common/src/test/java/eu/dnetlib/dhp/application/ArgumentApplicationParserTest.java
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/application/ArgumentApplicationParserTest.java
@@ -7,10 +7,10 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
-public class ArgumentApplicationParserTest {
+class ArgumentApplicationParserTest {
@Test
- public void testParseParameter() throws Exception {
+ void testParseParameter() throws Exception {
final String jsonConfiguration = IOUtils
.toString(
this.getClass().getResourceAsStream("/eu/dnetlib/application/parameters.json"));
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/HdfsSupportTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/HdfsSupportTest.java
index 870943816..fa721d5e5 100644
--- a/dhp-common/src/test/java/eu/dnetlib/dhp/common/HdfsSupportTest.java
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/common/HdfsSupportTest.java
@@ -21,13 +21,13 @@ public class HdfsSupportTest {
class Remove {
@Test
- public void shouldThrowARuntimeExceptionOnError() {
+ void shouldThrowARuntimeExceptionOnError() {
// when
assertThrows(RuntimeException.class, () -> HdfsSupport.remove(null, new Configuration()));
}
@Test
- public void shouldRemoveADirFromHDFS(@TempDir Path tempDir) {
+ void shouldRemoveADirFromHDFS(@TempDir Path tempDir) {
// when
HdfsSupport.remove(tempDir.toString(), new Configuration());
@@ -36,7 +36,7 @@ public class HdfsSupportTest {
}
@Test
- public void shouldRemoveAFileFromHDFS(@TempDir Path tempDir) throws IOException {
+ void shouldRemoveAFileFromHDFS(@TempDir Path tempDir) throws IOException {
// given
Path file = Files.createTempFile(tempDir, "p", "s");
@@ -52,13 +52,13 @@ public class HdfsSupportTest {
class ListFiles {
@Test
- public void shouldThrowARuntimeExceptionOnError() {
+ void shouldThrowARuntimeExceptionOnError() {
// when
assertThrows(RuntimeException.class, () -> HdfsSupport.listFiles(null, new Configuration()));
}
@Test
- public void shouldListFilesLocatedInPath(@TempDir Path tempDir) throws IOException {
+ void shouldListFilesLocatedInPath(@TempDir Path tempDir) throws IOException {
Path subDir1 = Files.createTempDirectory(tempDir, "list_me");
Path subDir2 = Files.createTempDirectory(tempDir, "list_me");
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/MdStoreClientTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/MdStoreClientTest.java
new file mode 100644
index 000000000..f87f6e313
--- /dev/null
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/common/MdStoreClientTest.java
@@ -0,0 +1,36 @@
+
+package eu.dnetlib.dhp.common;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class MdStoreClientTest {
+
+ // @Test
+ public void testMongoCollection() throws IOException {
+ final MdstoreClient client = new MdstoreClient("mongodb://localhost:27017", "mdstore");
+
+ final ObjectMapper mapper = new ObjectMapper();
+
+ final List infos = client.mdStoreWithTimestamp("ODF", "store", "cleaned");
+
+ infos.forEach(System.out::println);
+
+ final String s = mapper.writeValueAsString(infos);
+
+ Path fileName = Paths.get("/Users/sandro/mdstore_info.json");
+
+ // Writing into the file
+ Files.write(fileName, s.getBytes(StandardCharsets.UTF_8));
+
+ }
+}
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java
index 5ebd7213e..cb9ae2886 100644
--- a/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java
@@ -5,10 +5,10 @@ import static org.junit.jupiter.api.Assertions.*;
import org.junit.jupiter.api.Test;
-public class PacePersonTest {
+class PacePersonTest {
@Test
- public void pacePersonTest1() {
+ void pacePersonTest1() {
PacePerson p = new PacePerson("Artini, Michele", false);
assertEquals("Artini", p.getSurnameString());
@@ -17,7 +17,7 @@ public class PacePersonTest {
}
@Test
- public void pacePersonTest2() {
+ void pacePersonTest2() {
PacePerson p = new PacePerson("Michele G. Artini", false);
assertEquals("Artini, Michele G.", p.getNormalisedFullname());
assertEquals("Michele G", p.getNameString());
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/SparkSessionSupportTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/SparkSessionSupportTest.java
index 2f01c0863..8fa966c2f 100644
--- a/dhp-common/src/test/java/eu/dnetlib/dhp/common/SparkSessionSupportTest.java
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/common/SparkSessionSupportTest.java
@@ -18,7 +18,8 @@ public class SparkSessionSupportTest {
class RunWithSparkSession {
@Test
- public void shouldExecuteFunctionAndNotStopSparkSessionWhenSparkSessionIsNotManaged()
+ @SuppressWarnings("unchecked")
+ void shouldExecuteFunctionAndNotStopSparkSessionWhenSparkSessionIsNotManaged()
throws Exception {
// given
SparkSession spark = mock(SparkSession.class);
@@ -37,7 +38,8 @@ public class SparkSessionSupportTest {
}
@Test
- public void shouldExecuteFunctionAndStopSparkSessionWhenSparkSessionIsManaged()
+ @SuppressWarnings("unchecked")
+ void shouldExecuteFunctionAndStopSparkSessionWhenSparkSessionIsManaged()
throws Exception {
// given
SparkSession spark = mock(SparkSession.class);
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java
deleted file mode 100644
index 9ae9c33c2..000000000
--- a/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-@Disabled
-public class ZenodoAPIClientTest {
-
- private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
- private final String ACCESS_TOKEN = "";
-
- private final String CONCEPT_REC_ID = "657113";
-
- private final String depositionId = "674915";
-
- @Test
- public void testUploadOldDeposition() throws IOException, MissingConceptDoiException {
- ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
- ACCESS_TOKEN);
- Assertions.assertEquals(200, client.uploadOpenDeposition(depositionId));
-
- File file = new File(getClass()
- .getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
- .getPath());
-
- InputStream is = new FileInputStream(file);
-
- Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length()));
-
- String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
-
- Assertions.assertEquals(200, client.sendMretadata(metadata));
-
- Assertions.assertEquals(202, client.publish());
-
- }
-
- @Test
- public void testNewDeposition() throws IOException {
-
- ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
- ACCESS_TOKEN);
- Assertions.assertEquals(201, client.newDeposition());
-
- File file = new File(getClass()
- .getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
- .getPath());
-
- InputStream is = new FileInputStream(file);
-
- Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length()));
-
- String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
-
- Assertions.assertEquals(200, client.sendMretadata(metadata));
-
- Assertions.assertEquals(202, client.publish());
-
- }
-
- @Test
- public void testNewVersionNewName() throws IOException, MissingConceptDoiException {
-
- ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
- ACCESS_TOKEN);
-
- Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID));
-
- File file = new File(getClass()
- .getResource("/eu/dnetlib/dhp/common/api/newVersion")
- .getPath());
-
- InputStream is = new FileInputStream(file);
-
- Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition", file.length()));
-
- Assertions.assertEquals(202, client.publish());
-
- }
-
- @Test
- public void testNewVersionOldName() throws IOException, MissingConceptDoiException {
-
- ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
- ACCESS_TOKEN);
-
- Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID));
-
- File file = new File(getClass()
- .getResource("/eu/dnetlib/dhp/common/api/newVersion2")
- .getPath());
-
- InputStream is = new FileInputStream(file);
-
- Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition", file.length()));
-
- Assertions.assertEquals(202, client.publish());
-
- }
-
-}
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/model/mdstore/MetadataRecordTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/model/mdstore/MetadataRecordTest.java
deleted file mode 100644
index cb4d0ab50..000000000
--- a/dhp-common/src/test/java/eu/dnetlib/dhp/model/mdstore/MetadataRecordTest.java
+++ /dev/null
@@ -1,16 +0,0 @@
-
-package eu.dnetlib.dhp.model.mdstore;
-
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import org.junit.jupiter.api.Test;
-
-public class MetadataRecordTest {
-
- @Test
- public void getTimestamp() {
-
- MetadataRecord r = new MetadataRecord();
- assertTrue(r.getDateOfCollection() > 0);
- }
-}
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/GridCleaningRuleTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/GridCleaningRuleTest.java
new file mode 100644
index 000000000..1b9163d46
--- /dev/null
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/GridCleaningRuleTest.java
@@ -0,0 +1,18 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+class GridCleaningRuleTest {
+
+ @Test
+ void testCleaning() {
+ assertEquals("grid.493784.5", GridCleaningRule.clean("grid.493784.5"));
+ assertEquals("grid.493784.5x", GridCleaningRule.clean("grid.493784.5x"));
+ assertEquals("grid.493784.5x", GridCleaningRule.clean("493784.5x"));
+ assertEquals("", GridCleaningRule.clean("493x784.5x"));
+ }
+
+}
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/ISNICleaningRuleTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/ISNICleaningRuleTest.java
new file mode 100644
index 000000000..e51d1e05c
--- /dev/null
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/ISNICleaningRuleTest.java
@@ -0,0 +1,19 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+class ISNICleaningRuleTest {
+
+ @Test
+ void testCleaning() {
+ assertEquals("0000000463436020", ISNICleaningRule.clean("0000 0004 6343 6020"));
+ assertEquals("0000000463436020", ISNICleaningRule.clean("0000000463436020"));
+ assertEquals("", ISNICleaningRule.clean("Q30256598"));
+ assertEquals("0000000493403529", ISNICleaningRule.clean("ISNI:0000000493403529"));
+ assertEquals("000000008614884X", ISNICleaningRule.clean("0000 0000 8614 884X"));
+ }
+
+}
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtilsTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtilsTest.java
new file mode 100644
index 000000000..9111ac2df
--- /dev/null
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtilsTest.java
@@ -0,0 +1,212 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.Dataset;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Publication;
+import eu.dnetlib.dhp.schema.oaf.Result;
+import me.xuender.unidecode.Unidecode;
+
+class OafMapperUtilsTest {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ @Test
+ public void testUnidecode() {
+
+ assertEquals("Liu Ben Mu hiruzuSen tawa", Unidecode.decode("六本木ヒルズ森タワ"));
+ assertEquals("Nan Wu A Mi Tuo Fo", Unidecode.decode("南无阿弥陀佛"));
+ assertEquals("Yi Tiao Hui Zou Lu De Yu", Unidecode.decode("一条会走路的鱼"));
+ assertEquals("amidaniyorai", Unidecode.decode("あみだにょらい"));
+ assertEquals("T`owrk`iayi", Unidecode.decode("Թուրքիայի"));
+ assertEquals("Obzor tematiki", Unidecode.decode("Обзор тематики"));
+ assertEquals("GERMANSKIE IaZYKI", Unidecode.decode("ГЕРМАНСКИЕ ЯЗЫКИ"));
+ assertEquals("Diereunese tes ikanopoieses", Unidecode.decode("Διερεύνηση της ικανοποίησης"));
+ assertEquals("lqDy l'wly@", Unidecode.decode("القضايا الأولية"));
+ assertEquals("abc def ghi", Unidecode.decode("abc def ghi"));
+ }
+
+ @Test
+ void testDateValidation() {
+
+ assertNotNull(GraphCleaningFunctions.cleanDate("2016-05-07T12:41:19.202Z "));
+ assertNotNull(GraphCleaningFunctions.cleanDate("2020-09-10 11:08:52 "));
+ assertNotNull(GraphCleaningFunctions.cleanDate(" 2016-04-05"));
+
+ assertEquals("2016-04-05", GraphCleaningFunctions.cleanDate("2016 Apr 05"));
+
+ assertEquals("2009-05-08", GraphCleaningFunctions.cleanDate("May 8, 2009 5:57:51 PM"));
+ assertEquals("1970-10-07", GraphCleaningFunctions.cleanDate("oct 7, 1970"));
+ assertEquals("1970-10-07", GraphCleaningFunctions.cleanDate("oct 7, '70"));
+ assertEquals("1970-10-07", GraphCleaningFunctions.cleanDate("oct. 7, 1970"));
+ assertEquals("1970-10-07", GraphCleaningFunctions.cleanDate("oct. 7, 70"));
+ assertEquals("2006-01-02", GraphCleaningFunctions.cleanDate("Mon Jan 2 15:04:05 2006"));
+ assertEquals("2006-01-02", GraphCleaningFunctions.cleanDate("Mon Jan 2 15:04:05 MST 2006"));
+ assertEquals("2006-01-02", GraphCleaningFunctions.cleanDate("Mon Jan 02 15:04:05 -0700 2006"));
+ assertEquals("2006-01-02", GraphCleaningFunctions.cleanDate("Monday, 02-Jan-06 15:04:05 MST"));
+ assertEquals("2006-01-02", GraphCleaningFunctions.cleanDate("Mon, 02 Jan 2006 15:04:05 MST"));
+ assertEquals("2017-07-11", GraphCleaningFunctions.cleanDate("Tue, 11 Jul 2017 16:28:13 +0200 (CEST)"));
+ assertEquals("2006-01-02", GraphCleaningFunctions.cleanDate("Mon, 02 Jan 2006 15:04:05 -0700"));
+ assertEquals("2018-01-04", GraphCleaningFunctions.cleanDate("Thu, 4 Jan 2018 17:53:36 +0000"));
+ assertEquals("2015-08-10", GraphCleaningFunctions.cleanDate("Mon Aug 10 15:44:11 UTC+0100 2015"));
+ assertEquals(
+ "2015-07-03",
+ GraphCleaningFunctions.cleanDate("Fri Jul 03 2015 18:04:07 GMT+0100 (GMT Daylight Time)"));
+ assertEquals("2012-09-17", GraphCleaningFunctions.cleanDate("September 17, 2012 10:09am"));
+ assertEquals("2012-09-17", GraphCleaningFunctions.cleanDate("September 17, 2012 at 10:09am PST-08"));
+ assertEquals("2012-09-17", GraphCleaningFunctions.cleanDate("September 17, 2012, 10:10:09"));
+ assertEquals("1970-10-07", GraphCleaningFunctions.cleanDate("October 7, 1970"));
+ assertEquals("1970-10-07", GraphCleaningFunctions.cleanDate("October 7th, 1970"));
+ assertEquals("2006-02-12", GraphCleaningFunctions.cleanDate("12 Feb 2006, 19:17"));
+ assertEquals("2006-02-12", GraphCleaningFunctions.cleanDate("12 Feb 2006 19:17"));
+ assertEquals("1970-10-07", GraphCleaningFunctions.cleanDate("7 oct 70"));
+ assertEquals("1970-10-07", GraphCleaningFunctions.cleanDate("7 oct 1970"));
+ assertEquals("2013-02-03", GraphCleaningFunctions.cleanDate("03 February 2013"));
+ assertEquals("2013-07-01", GraphCleaningFunctions.cleanDate("1 July 2013"));
+ assertEquals("2013-02-03", GraphCleaningFunctions.cleanDate("2013-Feb-03"));
+ assertEquals("2014-03-31", GraphCleaningFunctions.cleanDate("3/31/2014"));
+ assertEquals("2014-03-31", GraphCleaningFunctions.cleanDate("03/31/2014"));
+ assertEquals("1971-08-21", GraphCleaningFunctions.cleanDate("08/21/71"));
+ assertEquals("1971-01-08", GraphCleaningFunctions.cleanDate("8/1/71"));
+ assertEquals("2014-08-04", GraphCleaningFunctions.cleanDate("4/8/2014 22:05"));
+ assertEquals("2014-08-04", GraphCleaningFunctions.cleanDate("04/08/2014 22:05"));
+ assertEquals("2014-08-04", GraphCleaningFunctions.cleanDate("4/8/14 22:05"));
+ assertEquals("2014-02-04", GraphCleaningFunctions.cleanDate("04/2/2014 03:00:51"));
+ assertEquals("1965-08-08", GraphCleaningFunctions.cleanDate("8/8/1965 12:00:00 AM"));
+ assertEquals("1965-08-08", GraphCleaningFunctions.cleanDate("8/8/1965 01:00:01 PM"));
+ assertEquals("1965-08-08", GraphCleaningFunctions.cleanDate("8/8/1965 01:00 PM"));
+ assertEquals("1965-08-08", GraphCleaningFunctions.cleanDate("8/8/1965 1:00 PM"));
+ assertEquals("1965-08-08", GraphCleaningFunctions.cleanDate("8/8/1965 12:00 AM"));
+ assertEquals("2014-02-04", GraphCleaningFunctions.cleanDate("4/02/2014 03:00:51"));
+ assertEquals("2012-03-19", GraphCleaningFunctions.cleanDate("03/19/2012 10:11:59"));
+ assertEquals("2012-03-19", GraphCleaningFunctions.cleanDate("03/19/2012 10:11:59.3186369"));
+ assertEquals("2014-03-31", GraphCleaningFunctions.cleanDate("2014/3/31"));
+ assertEquals("2014-03-31", GraphCleaningFunctions.cleanDate("2014/03/31"));
+ assertEquals("2014-04-08", GraphCleaningFunctions.cleanDate("2014/4/8 22:05"));
+ assertEquals("2014-04-08", GraphCleaningFunctions.cleanDate("2014/04/08 22:05"));
+ assertEquals("2014-04-02", GraphCleaningFunctions.cleanDate("2014/04/2 03:00:51"));
+ assertEquals("2014-04-02", GraphCleaningFunctions.cleanDate("2014/4/02 03:00:51"));
+ assertEquals("2012-03-19", GraphCleaningFunctions.cleanDate("2012/03/19 10:11:59"));
+ assertEquals("2012-03-19", GraphCleaningFunctions.cleanDate("2012/03/19 10:11:59.3186369"));
+ assertEquals("2014-04-08", GraphCleaningFunctions.cleanDate("2014年04月08日"));
+ assertEquals("2006-01-02", GraphCleaningFunctions.cleanDate("2006-01-02T15:04:05+0000"));
+ assertEquals("2009-08-13", GraphCleaningFunctions.cleanDate("2009-08-12T22:15:09-07:00"));
+ assertEquals("2009-08-12", GraphCleaningFunctions.cleanDate("2009-08-12T22:15:09"));
+ assertEquals("2014-04-26", GraphCleaningFunctions.cleanDate("2014-04-26 17:24:37.3186369"));
+ assertEquals("2012-08-03", GraphCleaningFunctions.cleanDate("2012-08-03 18:31:59.257000000"));
+ assertEquals("2014-04-26", GraphCleaningFunctions.cleanDate("2014-04-26 17:24:37.123"));
+ assertEquals("2013-04-01", GraphCleaningFunctions.cleanDate("2013-04-01 22:43"));
+ assertEquals("2013-04-01", GraphCleaningFunctions.cleanDate("2013-04-01 22:43:22"));
+ assertEquals("2014-12-16", GraphCleaningFunctions.cleanDate("2014-12-16 06:20:00 UTC"));
+ assertEquals("2014-12-16", GraphCleaningFunctions.cleanDate("2014-12-16 06:20:00 GMT"));
+ assertEquals("2014-04-26", GraphCleaningFunctions.cleanDate("2014-04-26 05:24:37 PM"));
+ assertEquals("2014-04-26", GraphCleaningFunctions.cleanDate("2014-04-26 13:13:43 +0800"));
+ assertEquals("2014-04-26", GraphCleaningFunctions.cleanDate("2014-04-26 13:13:43 +0800 +08"));
+ assertEquals("2014-04-26", GraphCleaningFunctions.cleanDate("2014-04-26 13:13:44 +09:00"));
+ assertEquals("2012-08-03", GraphCleaningFunctions.cleanDate("2012-08-03 18:31:59.257000000 +0000 UTC"));
+ assertEquals("2015-09-30", GraphCleaningFunctions.cleanDate("2015-09-30 18:48:56.35272715 +0000 UTC"));
+ assertEquals("2015-02-18", GraphCleaningFunctions.cleanDate("2015-02-18 00:12:00 +0000 GMT"));
+ assertEquals("2015-02-18", GraphCleaningFunctions.cleanDate("2015-02-18 00:12:00 +0000 UTC"));
+ assertEquals(
+ "2015-02-08", GraphCleaningFunctions.cleanDate("2015-02-08 03:02:00 +0300 MSK m=+0.000000001"));
+ assertEquals(
+ "2015-02-08", GraphCleaningFunctions.cleanDate("2015-02-08 03:02:00.001 +0300 MSK m=+0.000000001"));
+ assertEquals("2017-07-19", GraphCleaningFunctions.cleanDate("2017-07-19 03:21:51+00:00"));
+ assertEquals("2014-04-26", GraphCleaningFunctions.cleanDate("2014-04-26"));
+ assertEquals("2014-04-01", GraphCleaningFunctions.cleanDate("2014-04"));
+ assertEquals("2014-01-01", GraphCleaningFunctions.cleanDate("2014"));
+ assertEquals("2014-05-11", GraphCleaningFunctions.cleanDate("2014-05-11 08:20:13,787"));
+ assertEquals("2014-03-31", GraphCleaningFunctions.cleanDate("3.31.2014"));
+ assertEquals("2014-03-31", GraphCleaningFunctions.cleanDate("03.31.2014"));
+ assertEquals("1971-08-21", GraphCleaningFunctions.cleanDate("08.21.71"));
+ assertEquals("2014-03-01", GraphCleaningFunctions.cleanDate("2014.03"));
+ assertEquals("2014-03-30", GraphCleaningFunctions.cleanDate("2014.03.30"));
+ assertEquals("2014-06-01", GraphCleaningFunctions.cleanDate("20140601"));
+ assertEquals("2014-07-22", GraphCleaningFunctions.cleanDate("20140722105203"));
+ assertEquals("2012-03-19", GraphCleaningFunctions.cleanDate("1332151919"));
+ assertEquals("2013-11-12", GraphCleaningFunctions.cleanDate("1384216367189"));
+ assertEquals("2013-11-12", GraphCleaningFunctions.cleanDate("1384216367111222"));
+ assertEquals("2013-11-12", GraphCleaningFunctions.cleanDate("1384216367111222333"));
+
+ }
+
+ @Test
+ void testDate() {
+ final String date = GraphCleaningFunctions.cleanDate("23-FEB-1998");
+ assertNotNull(date);
+ System.out.println(date);
+ }
+
+ @Test
+ void testMergePubs() throws IOException {
+ Publication p1 = read("publication_1.json", Publication.class);
+ Publication p2 = read("publication_2.json", Publication.class);
+ Dataset d1 = read("dataset_1.json", Dataset.class);
+ Dataset d2 = read("dataset_2.json", Dataset.class);
+
+ assertEquals(1, p1.getCollectedfrom().size());
+ assertEquals(ModelConstants.CROSSREF_ID, p1.getCollectedfrom().get(0).getKey());
+ assertEquals(1, d2.getCollectedfrom().size());
+ assertFalse(cfId(d2.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
+
+ assertEquals(
+ ModelConstants.PUBLICATION_RESULTTYPE_CLASSID,
+ OafMapperUtils
+ .mergeResults(p1, d2)
+ .getResulttype()
+ .getClassid());
+
+ assertEquals(1, p2.getCollectedfrom().size());
+ assertFalse(cfId(p2.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
+ assertEquals(1, d1.getCollectedfrom().size());
+ assertTrue(cfId(d1.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
+
+ assertEquals(
+ ModelConstants.DATASET_RESULTTYPE_CLASSID,
+ OafMapperUtils
+ .mergeResults(p2, d1)
+ .getResulttype()
+ .getClassid());
+ }
+
+ @Test
+ void testDelegatedAuthority() throws IOException {
+ Dataset d1 = read("dataset_2.json", Dataset.class);
+ Dataset d2 = read("dataset_delegated.json", Dataset.class);
+
+ assertEquals(1, d2.getCollectedfrom().size());
+ assertTrue(cfId(d2.getCollectedfrom()).contains(ModelConstants.ZENODO_OD_ID));
+
+ Result res = OafMapperUtils.mergeResults(d1, d2);
+
+ assertEquals(d2, res);
+
+ System.out.println(OBJECT_MAPPER.writeValueAsString(res));
+
+ }
+
+ protected HashSet cfId(List collectedfrom) {
+ return collectedfrom.stream().map(KeyValue::getKey).collect(Collectors.toCollection(HashSet::new));
+ }
+
+ protected T read(String filename, Class clazz) throws IOException {
+ final String json = IOUtils.toString(getClass().getResourceAsStream(filename));
+ return OBJECT_MAPPER.readValue(json, clazz);
+ }
+
+}
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PICCleaningRuleTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PICCleaningRuleTest.java
new file mode 100644
index 000000000..3736033c3
--- /dev/null
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PICCleaningRuleTest.java
@@ -0,0 +1,19 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+class PICCleaningRuleTest {
+
+ @Test
+ void testCleaning() {
+ assertEquals("887624982", PICCleaningRule.clean("887624982"));
+ assertEquals("", PICCleaningRule.clean("887 624982"));
+ assertEquals("887624982", PICCleaningRule.clean(" 887624982 "));
+ assertEquals("887624982", PICCleaningRule.clean(" 887624982x "));
+ assertEquals("887624982", PICCleaningRule.clean(" 88762498200 "));
+ }
+
+}
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PmcCleaningRuleTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PmcCleaningRuleTest.java
new file mode 100644
index 000000000..e53ebae89
--- /dev/null
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PmcCleaningRuleTest.java
@@ -0,0 +1,19 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+class PmcCleaningRuleTest {
+
+ @Test
+ void testCleaning() {
+ assertEquals("PMC1234", PmcCleaningRule.clean("PMC1234"));
+ assertEquals("PMC1234", PmcCleaningRule.clean(" PMC1234"));
+ assertEquals("PMC12345678", PmcCleaningRule.clean("PMC12345678"));
+ assertEquals("PMC12345678", PmcCleaningRule.clean("PMC123456789"));
+ assertEquals("PMC12345678", PmcCleaningRule.clean("PMC 12345678"));
+ }
+
+}
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRuleTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRuleTest.java
new file mode 100644
index 000000000..295eac85f
--- /dev/null
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRuleTest.java
@@ -0,0 +1,24 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+class PmidCleaningRuleTest {
+
+ @Test
+ void testCleaning() {
+ // leading zeros are removed
+ assertEquals("1234", PmidCleaningRule.clean("01234"));
+ // tolerant to spaces in the middle
+ assertEquals("1234567", PmidCleaningRule.clean("0123 4567"));
+ // stop parsing at first not numerical char
+ assertEquals("123", PmidCleaningRule.clean("0123x4567"));
+ // invalid id leading to empty result
+ assertEquals("", PmidCleaningRule.clean("abc"));
+ // valid id with zeroes in the number
+ assertEquals("20794075", PmidCleaningRule.clean("20794075"));
+ }
+
+}
diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/RorCleaningRuleTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/RorCleaningRuleTest.java
new file mode 100644
index 000000000..5d5c03959
--- /dev/null
+++ b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/RorCleaningRuleTest.java
@@ -0,0 +1,17 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+class RorCleaningRuleTest {
+
+ @Test
+ void testCleaning() {
+ assertEquals("https://ror.org/05rpz9w55", RorCleaningRule.clean("https://ror.org/05rpz9w55"));
+ assertEquals("https://ror.org/05rpz9w55", RorCleaningRule.clean("05rpz9w55"));
+ assertEquals("", RorCleaningRule.clean("05rpz9w_55"));
+ }
+
+}
diff --git a/dhp-common/src/test/java/eu/dnetlib/message/MessageTest.java b/dhp-common/src/test/java/eu/dnetlib/message/MessageTest.java
deleted file mode 100644
index 442f7b5c2..000000000
--- a/dhp-common/src/test/java/eu/dnetlib/message/MessageTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-
-package eu.dnetlib.message;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.junit.jupiter.api.Test;
-
-public class MessageTest {
-
- @Test
- public void fromJsonTest() throws IOException {
- Message m = new Message();
- m.setWorkflowId("wId");
- m.setType(MessageType.ONGOING);
- m.setJobName("Collection");
- Map body = new HashMap<>();
- body.put("parsedItem", "300");
- body.put("ExecutionTime", "30s");
-
- m.setBody(body);
- System.out.println("m = " + m);
- Message m1 = Message.fromJson(m.toString());
- assertEquals(m1.getWorkflowId(), m.getWorkflowId());
- assertEquals(m1.getType(), m.getType());
- assertEquals(m1.getJobName(), m.getJobName());
-
- assertNotNull(m1.getBody());
- m1.getBody().keySet().forEach(it -> assertEquals(m1.getBody().get(it), m.getBody().get(it)));
- assertEquals(m1.getJobName(), m.getJobName());
- }
-
- @Test
- public void toStringTest() {
- final String expectedJson = "{\"workflowId\":\"wId\",\"jobName\":\"Collection\",\"type\":\"ONGOING\",\"body\":{\"ExecutionTime\":\"30s\",\"parsedItem\":\"300\"}}";
- Message m = new Message();
- m.setWorkflowId("wId");
- m.setType(MessageType.ONGOING);
- m.setJobName("Collection");
- Map body = new HashMap<>();
- body.put("parsedItem", "300");
- body.put("ExecutionTime", "30s");
-
- m.setBody(body);
-
- assertEquals(expectedJson, m.toString());
- }
-}
diff --git a/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java b/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java
new file mode 100644
index 000000000..c0a8d6927
--- /dev/null
+++ b/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java
@@ -0,0 +1,114 @@
+
+package eu.dnetlib.oa.merge;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.List;
+import java.util.Objects;
+
+import org.junit.jupiter.api.Test;
+import org.junit.platform.commons.util.StringUtils;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.oa.merge.AuthorMerger;
+import eu.dnetlib.dhp.schema.oaf.Author;
+
+public class AuthorMergerTest {
+
+ @Test
+ public void testEnrcichAuthor() throws Exception {
+ final ObjectMapper mapper = new ObjectMapper();
+
+ BufferedReader pr = new BufferedReader(new InputStreamReader(
+ Objects
+ .requireNonNull(
+ AuthorMergerTest.class
+ .getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_publication_sample.json"))));
+ BufferedReader or = new BufferedReader(new InputStreamReader(
+ Objects
+ .requireNonNull(
+ AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_orcid_sample.json"))));
+
+ TypeReference