optimized some dedup functions

dhp-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java

@@ -49,14 +49,14 @@ public abstract class AbstractPaceFunctions {
 	protected static Set<String> ngramBlacklist = loadFromClasspath("/eu/dnetlib/pace/config/ngram_blacklist.txt");
 
 	// html regex for normalization
-	public final String HTML_REGEX = "<[^>]*>";
+	public final Pattern HTML_REGEX = Pattern.compile("<[^>]*>");
 
 	private static final String alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ";
 	private static final String aliases_from = "⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎àáâäæãåāèéêëēėęəîïíīįìôöòóœøōõûüùúūßśšłžźżçćčñń";
 	private static final String aliases_to = "0123456789+-=()n0123456789+-=()aaaaaaaaeeeeeeeeiiiiiioooooooouuuuussslzzzcccnn";
 
 	// doi prefix for normalization
-	public final String DOI_PREFIX = "(https?:\\/\\/dx\\.doi\\.org\\/)|(doi:)";
+	public final Pattern DOI_PREFIX = Pattern.compile("(https?:\\/\\/dx\\.doi\\.org\\/)|(doi:)");
 
 	private Pattern numberPattern = Pattern.compile("-?\\d+(\\.\\d+)?");
 
@@ -67,8 +67,7 @@ public abstract class AbstractPaceFunctions {
 	}
 
 	protected String cleanup(final String s) {
-
+		final String s1 = HTML_REGEX.matcher(s).replaceAll( "");
-		final String s1 = s.replaceAll(HTML_REGEX, "");
 		final String s2 = unicodeNormalization(s1.toLowerCase());
 		final String s3 = nfd(s2);
 		final String s4 = fixXML(s3);
@@ -302,7 +301,7 @@ public abstract class AbstractPaceFunctions {
 	}
 
 	public String normalizePid(String pid) {
-		return pid.toLowerCase().replaceAll(DOI_PREFIX, "");
+		return DOI_PREFIX.matcher(pid.toLowerCase()).replaceAll("");
 	}
 
 	// get the list of keywords into the input string

dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/JsonListMatch.java

@@ -6,6 +6,9 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.Option;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -15,6 +18,7 @@ import eu.dnetlib.pace.config.Config;
 import eu.dnetlib.pace.tree.support.AbstractListComparator;
 import eu.dnetlib.pace.tree.support.ComparatorClass;
 import eu.dnetlib.pace.util.MapDocumentUtil;
+import com.jayway.jsonpath.JsonPath;
 
 @ComparatorClass("jsonListMatch")
 public class JsonListMatch extends AbstractListComparator {
@@ -59,11 +63,11 @@ public class JsonListMatch extends AbstractListComparator {
 
 		StringBuilder st = new StringBuilder(); // to build the string used for comparisons basing on the jpath into
 												// parameters
-
+		final DocumentContext documentContext = JsonPath.using(Configuration.defaultConfiguration().addOptions(Option.SUPPRESS_EXCEPTIONS)).parse(json);
 		// for each path in the param list
 		for (String key : params.keySet().stream().filter(k -> k.contains("jpath")).collect(Collectors.toList())) {
 			String path = params.get(key);
-			String value = MapDocumentUtil.getJPathString(path, json);
+			String value = MapDocumentUtil.getJPathString(path, documentContext);
 			if (value == null || value.isEmpty())
 				value = "";
 			st.append(value);

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java

@@ -18,12 +18,12 @@ import org.xml.sax.SAXException;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.application.dedup.log.DedupLogModel;
 import eu.dnetlib.dhp.application.dedup.log.DedupLogWriter;
-import eu.dnetlib.dhp.oa.dedup.model.SparkDedupConfig;
 import eu.dnetlib.dhp.schema.oaf.Relation;
 import eu.dnetlib.dhp.utils.ISLookupClientFactory;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
 import eu.dnetlib.pace.config.DedupConfig;
+import eu.dnetlib.pace.model.SparkDedupConfig;
 
 public class SparkCreateSimRels extends AbstractSparkAction {
 
@@ -67,6 +67,7 @@ public class SparkCreateSimRels extends AbstractSparkAction {
 		log.info("isLookUpUrl:   '{}'", isLookUpUrl);
 		log.info("actionSetId:   '{}'", actionSetId);
 		log.info("workingPath:   '{}'", workingPath);
+
 		final String dfLogPath = parser.get("dataframeLog");
 		final String runTag = Optional.ofNullable(parser.get("runTAG")).orElse("UNKNOWN");
 
@@ -90,10 +91,10 @@ public class SparkCreateSimRels extends AbstractSparkAction {
 				.textFile(DedupUtility.createEntityPath(graphBasePath, subEntity))
 				.transform(sparkConfig.modelExtractor()) // Extract fields from input json column according to model
 				// definition
-					.transform(sparkConfig.generateAndProcessClustersWithJoins()) // generate <key,block> pairs according to
+				.transform(sparkConfig.generateClustersWithDFAPI()) // generate <key,block> pairs according to
 				// filters, clusters, and model
 				// definition
-					// .transform(sparkConfig.processClusters()) // process blocks and emits <from,to> pairs of found
+				.transform(sparkConfig.processClusters()) // process blocks and emits <from,to> pairs of found
 				// similarities
 				.map(
 					(MapFunction<Row, Relation>) t -> DedupUtility