D-Net
/
dnet-dedup
5
1
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

bug fix in the AuthorMatch, implementation of the concat function in the model creation with jpath query

This commit is contained in:
miconis 2022-03-09 12:53:09 +01:00
parent 613b32fcf7
commit cb72ce0a22
6 changed files with 35 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dnet-dedup-test/src/test/java/eu/dnetlib/pace/DedupLocalTest.java
View File

File diff suppressed because one or more lines are too long

5
dnet-dedup-test/src/test/resources/eu/dnetlib/pace/config/pub.new.tree.conf.json
View File

@ -178,6 +178,7 @@
"params": {
"surname_th": 0.75,
"fullname_th": 0.75,
"size_th": 20,
"mode": "surname"
}
}
@ -215,8 +216,8 @@
},
{
"name": "title",
"type": "String",
"path": "$.title[?(@.qualifier.classid == 'main title')].value",
"type": "StringConcat",
"path": "$.title[?(@.qualifier.classid == 'main title')].value|||$.title[?(@.qualifier.classid == 'subtitle')].value",
"length": 250,
"size": 5
},

35
dnet-dedup-test/src/test/resources/eu/dnetlib/pace/examples/publications.to.fix.json
View File

File diff suppressed because one or more lines are too long

2
dnet-pace-core/src/main/java/eu/dnetlib/pace/config/Type.java
View File

@ -1,5 +1,5 @@
package eu.dnetlib.pace.config;
public enum Type {
String, Int, List, JSON, URL
String, Int, List, JSON, URL, StringConcat
}

6
dnet-pace-core/src/main/java/eu/dnetlib/pace/tree/AuthorsMatch.java
View File

@ -77,11 +77,13 @@ public class AuthorsMatch extends AbstractComparator {
//one person is inaccurate
if (p1.isAccurate() ^ p2.isAccurate()) {
//prepare data
//data for the accurate person
String name = normalization(p1.isAccurate()? p1.getNormalisedFirstName() : p2.getNormalisedFirstName());
String surname = normalization(p1.isAccurate()? p2.getNormalisedSurname() : p2.getNormalisedSurname());
String surname = normalization(p1.isAccurate()? p1.getNormalisedSurname() : p2.getNormalisedSurname());
//data for the inaccurate person
String fullname = normalization(
p1.isAccurate() ? ((p1.getNormalisedFullname().isEmpty()) ? p1.getOriginal() : p1.getNormalisedFullname()) : (p2.getNormalisedFullname().isEmpty() ? p2.getOriginal() : p2.getNormalisedFullname())
p1.isAccurate() ? ((p2.getNormalisedFullname().isEmpty()) ? p2.getOriginal() : p2.getNormalisedFullname()) : (p1.getNormalisedFullname().isEmpty() ? p1.getOriginal() : p1.getNormalisedFullname())
);
if (fullname.contains(surname)) {

12
dnet-pace-core/src/main/java/eu/dnetlib/pace/util/MapDocumentUtil.java
View File

@ -15,6 +15,7 @@ import net.minidev.json.JSONArray;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
public class MapDocumentUtil {
@ -45,6 +46,17 @@ public class MapDocumentUtil {
.forEach(fi::add);
stringField.put(fdef.getName(), fi);
break;
case StringConcat:
String[] jpaths = fdef.getPath().split("\\|\\|\\|");
stringField.put(
fdef.getName(),
new FieldValueImpl(Type.String,
fdef.getName(),
truncateValue(Arrays.stream(jpaths).map(jpath -> getJPathString(jpath, json)).collect(Collectors.joining(" ")),
fdef.getLength())
)
);
break;
}
});
m.setFieldMap(stringField);