sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

implementation of the support for authors deduplication: cosinesimilarity comparator and double array json parser

This commit is contained in:
Michele De Bonis 2023-04-17 11:06:27 +02:00
parent 297eb207a5
commit cb595c87bb
13 changed files with 197 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dnet-pace-core/src/main/java/eu/dnetlib/pace/config/Type.java
View File

@ -1,5 +1,5 @@
package eu.dnetlib.pace.config;
public enum Type {
String, Int, List, JSON, URL, StringConcat
String, Int, List, JSON, URL, StringConcat, DoubleArray
}

2
dnet-pace-core/src/main/java/eu/dnetlib/pace/model/FieldValue.java
View File

@ -20,4 +20,6 @@ public interface FieldValue extends Field {
*/
public void setValue(final Object value);
public double[] doubleArrayValue();
}

6
dnet-pace-core/src/main/java/eu/dnetlib/pace/model/FieldValueImpl.java
View File

@ -60,6 +60,8 @@ public class FieldValueImpl extends AbstractField implements FieldValue {
case URL:
String str = value.toString();
return StringUtils.isBlank(str) || !isValidURL(str);
case DoubleArray:
return doubleArrayValue().length==0;
default:
return true;
}
@ -116,6 +118,10 @@ public class FieldValueImpl extends AbstractField implements FieldValue {
// }
}
public double[] doubleArrayValue() {
return (double[])getValue();
}
/*
* (non-Javadoc)
*

2
dnet-pace-core/src/main/java/eu/dnetlib/pace/tree/AuthorsMatch.java
View File

@ -52,7 +52,7 @@ public class AuthorsMatch extends AbstractComparator {
if (a.isEmpty() || b.isEmpty())
return -1;
if (((FieldList) a).size() > SIZE_THRESHOLD || ((FieldList) a).size() > SIZE_THRESHOLD)
if (((FieldList) a).size() > SIZE_THRESHOLD || ((FieldList) b).size() > SIZE_THRESHOLD)
return 1.0;
List<Person> aList = ((FieldList) a).stringList().stream().map(author -> new Person(author, false)).collect(Collectors.toList());

53
dnet-pace-core/src/main/java/eu/dnetlib/pace/tree/CosineSimilarity.java Normal file
View File

@ -0,0 +1,53 @@
package eu.dnetlib.pace.tree;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldList;
import eu.dnetlib.pace.model.FieldValueImpl;
import eu.dnetlib.pace.model.Person;
import eu.dnetlib.pace.tree.support.AbstractComparator;
import eu.dnetlib.pace.tree.support.ComparatorClass;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@ComparatorClass("cosineSimilarity")
public class CosineSimilarity extends AbstractComparator {
Map<String, String> params;
public CosineSimilarity(Map<String,String> params) {
super(params);
}
@Override
public double compare(final Field a, final Field b, final Config conf) {
if (a.isEmpty() || b.isEmpty())
return -1;
double[] aVector = ((FieldValueImpl) a).doubleArrayValue();
double[] bVector = ((FieldValueImpl) b).doubleArrayValue();
return cosineSimilarity(aVector, bVector);
}
double cosineSimilarity(double[] a, double[] b) {
double dotProduct = 0;
double normASum = 0;
double normBSum = 0;
for(int i = 0; i < a.length; i ++) {
dotProduct += a[i] * b[i];
normASum += a[i] * a[i];
normBSum += b[i] * b[i];
}
double eucledianDist = Math.sqrt(normASum) * Math.sqrt(normBSum);
return dotProduct / eucledianDist;
}
}

9
dnet-pace-core/src/main/java/eu/dnetlib/pace/tree/StringListMatch.java
View File

@ -19,9 +19,13 @@ public class StringListMatch extends AbstractComparator {
private static final Log log = LogFactory.getLog(StringListMatch.class);
private Map<String, String> params;
final private String TYPE; //percentage or count
public StringListMatch(final Map<String, String> params) {
super(params);
this.params = params;
TYPE = params.getOrDefault("type", "percentage");
}
@Override
@ -31,7 +35,7 @@ public class StringListMatch extends AbstractComparator {
final Set<String> pb = new HashSet<>(((FieldList) b).stringList());
if (pa.isEmpty() || pb.isEmpty()) {
return -1; //return undefined if one of the two lists of pids is empty
return -1; //return undefined if one of the two lists is empty
}
int incommon = Sets.intersection(pa, pb).size();
@ -41,7 +45,10 @@ public class StringListMatch extends AbstractComparator {
return 0.0;
}
if(TYPE.equals("percentage"))
return (double)incommon / (incommon + simDiff);
else
return incommon;
}
}

1
dnet-pace-core/src/main/java/eu/dnetlib/pace/util/BlockProcessorForTesting.java
View File

@ -244,6 +244,5 @@ public class BlockProcessorForTesting {
final String type = dedupConf.getWf().getEntityType();
context.emit(type, from, to);
context.emit(type, to, from);
}
}

38
dnet-pace-core/src/main/java/eu/dnetlib/pace/util/MapDocumentUtil.java
View File

@ -7,12 +7,10 @@ import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
import eu.dnetlib.pace.config.DedupConfig;
import eu.dnetlib.pace.config.Type;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldListImpl;
import eu.dnetlib.pace.model.FieldValueImpl;
import eu.dnetlib.pace.model.MapDocument;
import eu.dnetlib.pace.model.*;
import net.minidev.json.JSONArray;
import java.math.BigDecimal;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@ -46,6 +44,14 @@ public class MapDocumentUtil {
.forEach(fi::add);
stringField.put(fdef.getName(), fi);
break;
case DoubleArray:
stringField.put(
fdef.getName(),
new FieldValueImpl(Type.DoubleArray,
fdef.getName(),
getJPathArray(fdef.getPath(), json))
);
break;
case StringConcat:
String[] jpaths = fdef.getPath().split("\\|\\|\\|");
stringField.put(
@ -115,6 +121,30 @@ public class MapDocumentUtil {
}
}
public static double[] getJPathArray(final String jsonPath, final String json) {
try {
Object o = JsonPath.read(json, jsonPath);
if (o instanceof double[])
return (double[]) o;
if (o instanceof JSONArray) {
Object[] objects = ((JSONArray) o).toArray();
double[] array = new double[objects.length];
for (int i = 0; i < objects.length; i++) {
if (objects[i] instanceof BigDecimal)
array[i] = ((BigDecimal)objects[i]).doubleValue();
else
array[i] = (double) objects[i];
}
return array;
}
return new double[0];
}
catch (Exception e) {
e.printStackTrace();
return new double[0];
}
}
public static String truncateValue(String value, int length) {
if (value == null)

4
dnet-pace-core/src/test/java/eu/dnetlib/pace/AbstractPaceTest.java
View File

@ -36,6 +36,10 @@ public abstract class AbstractPaceTest extends AbstractPaceFunctions {
return new FieldValueImpl(Type.URL, "url", s);
}
protected Field array(final double[] a) {
return new FieldValueImpl(Type.DoubleArray, "array", a);
}
protected Field createFieldList(List<String> strings, String fieldName){
List<FieldValueImpl> fieldValueStream = strings.stream().map(s -> new FieldValueImpl(Type.String, fieldName, s)).collect(Collectors.toList());

15
dnet-pace-core/src/test/java/eu/dnetlib/pace/comparators/ComparatorTest.java
View File

@ -2,7 +2,9 @@ package eu.dnetlib.pace.comparators;
import eu.dnetlib.pace.AbstractPaceTest;
import eu.dnetlib.pace.clustering.NGramUtils;
import eu.dnetlib.pace.config.Type;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldValueImpl;
import eu.dnetlib.pace.tree.*;
import eu.dnetlib.pace.config.DedupConfig;
@ -284,5 +286,18 @@ public class ComparatorTest extends AbstractPaceTest {
}
@Test
public void cosineSimilarity() {
CosineSimilarity cosineSimilarity = new CosineSimilarity(params);
Field a = new FieldValueImpl(Type.DoubleArray, "array", new double[]{1,2,3});
Field b = new FieldValueImpl(Type.DoubleArray, "array", new double[]{1,2,3});
double compare = cosineSimilarity.compare(a, b, conf);
System.out.println("compare = " + compare);
}
}

12
dnet-pace-core/src/test/java/eu/dnetlib/pace/config/ConfigTest.java
View File

@ -7,6 +7,7 @@ import eu.dnetlib.pace.clustering.ClusteringClass;
import eu.dnetlib.pace.clustering.ClusteringCombiner;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldList;
import eu.dnetlib.pace.model.FieldValue;
import eu.dnetlib.pace.model.MapDocument;
import eu.dnetlib.pace.tree.JsonListMatch;
import eu.dnetlib.pace.tree.support.AggType;
@ -20,10 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
@ -104,15 +102,15 @@ public class ConfigTest extends AbstractPaceTest {
}
@Test
public void asMapDocumentTest2() {
public void authorAsMapDocument() {
DedupConfig dedupConf = DedupConfig.load(readFromClasspath("author.test.conf.json"));
DedupConfig dedupConf = DedupConfig.load(readFromClasspath("author.fdup.conf.json"));
final String json = readFromClasspath("author.json");
final MapDocument mapDocument = MapDocumentUtil.asMapDocumentWithJPath(dedupConf, json);
System.out.println("mapDocument = " + mapDocument.getFieldMap().get("coauthors").stringValue());
System.out.println("mapDocument = " + Arrays.toString(((FieldValue) mapDocument.getFieldMap().get("topics")).doubleArrayValue()));
}

96
dnet-pace-core/src/test/resources/eu/dnetlib/pace/config/author.test.conf.json → dnet-pace-core/src/test/resources/eu/dnetlib/pace/config/author.fdup.conf.json
View File

@ -29,71 +29,103 @@
},
"pace": {
"clustering" : [
{ "name" : "personClustering", "fields" : [ "fullname" ], "params" : {} },
{ "name" : "personHash", "fields" : [ "fullname" ], "params" : {} }
{ "name" : "lnfi", "fields" : [ "fullname" ], "params" : {} }
],
"decisionTree": {
"start": {
"fields": [
{
"field": "year",
"comparator": "numbersComparator",
"weight": 1,
"countIfUndefined": "false",
"field": "orcid",
"comparator": "exactMatch",
"weight": 1.0,
"countIfUndefined": "true",
"params": {}
}
],
"threshold": 50,
"threshold": 1.0,
"aggregation": "MAX",
"positive": "NO_MATCH",
"negative": "surnames",
"undefined": "surnames",
"positive": "MATCH",
"negative": "NO_MATCH",
"undefined": "orcids",
"ignoreUndefined": "true"
},
"surnames": {
"orcids": {
"fields": [
{
"field": "orcids",
"comparator": "stringListMatch",
"weight": 1.0,
"countIfUndefined": "true",
"params": {"type": "count"}
}
],
"threshold": 3.0,
"aggregation": "MAX",
"positive": "MATCH",
"negative": "coauthors",
"undefined": "coauthors",
"ignoreUndefined": "true"
},
"coauthors": {
"fields": [
{
"field": "coauthors",
"comparator": "authorsMatch",
"weight": 1.0,
"countIfUndefined": "false",
"params": {
"surname_th": 0.75,
"fullname_th": 0.75,
"size_th": 20,
"mode": "surname"
}
"countIfUndefined": "true",
"params": {"type": "count"}
}
],
"threshold": 0.6,
"threshold": 1.0,
"aggregation": "MAX",
"positive": "topicsMatch",
"negative": "NO_MATCH",
"undefined": "topicsMatch",
"ignoreUndefined": "true"
},
"topicsMatch": {
"fields": [
{
"field": "topics",
"comparator": "cosineSimilarity",
"weight": 1.0,
"countIfUndefined": "true",
"params": {}
}
],
"threshold": 1.0,
"aggregation": "MAX",
"positive": "MATCH",
"negative": "NO_MATCH",
"undefined": "MATCH",
"ignoreUndefined": "true"
"undefined": "NO_MATCH",
"ignoreUndefined": "false"
}
},
"model": [
{
"name": "topics",
"type": "DoubleArray",
"path": "$.topics"
},
{
"name": "fullname",
"type": "String",
"path": "$.name"
"path": "$.fullname"
},
{
"name": "orcid",
"type": "String",
"path": "$.orcid"
},
{
"name": "coauthors",
"type": "List",
"path": "$.coauthors[*].name",
"size": 200
"path": "$.coAuthors[*].fullname"
},
{
"name": "year",
"type": "String",
"path": "$.publication.year"
},
{
"name": "title",
"type": "String",
"path": "$.publication.title"
"name": "orcids",
"type": "List",
"path": "$.coAuthors[*].orcid"
}
],
"blacklists": {},

2
dnet-pace-core/src/test/resources/eu/dnetlib/pace/config/author.json
View File

@ -1 +1 @@
{"id": "f3389e7c8af1d806c06e2ab51f28a4b4", "name": "Aczél, János", "shortname": "Aczél, J.", "pid": "aczel.janos", "coauthors": [], "publication": {"year": "1955", "title": "L\\\"osung der Vektor-Funktionalgleichung der homogenen und inhomogenen $n$-dimensionalen einparametrigen ``Translation'' der erzeugenden Funktion von Kettenreaktionen und des station\\\"aren und nichtstation\\\"aren Bewegungsintegrals", "venue": "Acta Math. Acad. Sci. Hung. 6, 131-140 (1955)."}}
{"fullname":"Zaragoza, Maria Cleofé","firstname":"Maria Cleofé","lastname":"Zaragoza","coAuthors":[{"fullname":"Cambras, Trinitat","lastname":"Cambras","firstname":"Trinitat","orcid":"0000-0002-9009-4690"},{"fullname":"Castro-Marrero, Jesús","lastname":"Castro-Marrero","firstname":"Jesús","orcid":""},{"fullname":"Díez-Noguera, Antoni","lastname":"Díez-Noguera","firstname":"Antoni","orcid":""},{"fullname":"Alegre, José","lastname":"Alegre","firstname":"José","orcid":"0000-0002-7582-7585"}],"topics":[0.9522090839562252,0.04779091604377485],"orcid":"0000-0002-9797-0219","id":"author::1a10826c83c7f9f0dcebe7df05e37a2a","pubId":"50|pmid________::db7fd19db5a620eafad40cfb97f9690d"}