D-Net
/
dnet-dedup
5
1
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

added DiffPatchMatch utility. Resumed commented tests!

This commit is contained in:
Claudio Atzori 2018-10-31 10:49:11 +01:00
parent 7c59c3ebf0
commit 9f513352fb
13 changed files with 6058 additions and 1109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dnet-dedup-test/src/main/java/eu/dnetlib/SparkTest.java
View File

@ -22,22 +22,24 @@ import scala.Tuple2;
import java.io.IOException; import java.io.IOException;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.URL;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
public class SparkTest { public class SparkTest {
public static SparkCounter counter ; public static SparkCounter counter ;
private static final Log log = LogFactory.getLog(SparkTest.class); private static final Log log = LogFactory.getLog(SparkTest.class);
public static void main(String[] args) { public static void main(String[] args) {
final JavaSparkContext context = new JavaSparkContext(new SparkConf().setAppName("Deduplication").setMaster("local[*]")); final JavaSparkContext context = new JavaSparkContext(new SparkConf().setAppName("Deduplication").setMaster("local[*]"));
final JavaRDD<String> dataRDD = context.textFile("file:///Users/miconis/Downloads/dumps/organizations_sample.json");
final URL dataset = SparkTest.class.getResource("/eu/dnetlib/pace/orgs.test.json");
final JavaRDD<String> dataRDD = context.textFile(dataset.getPath());
counter = new SparkCounter(context); counter = new SparkCounter(context);
//read the configuration from the classpath //read the configuration from the classpath
final DedupConfig config = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf")); final DedupConfig config = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.test.pace.conf"));
BlockProcessor.constructAccumulator(config); BlockProcessor.constructAccumulator(config);
BlockProcessor.accumulators.forEach(acc -> { BlockProcessor.accumulators.forEach(acc -> {

30
dnet-dedup-test/src/main/resources/eu/dnetlib/pace/organization.beta.pace.conf Normal file
View File

@ -0,0 +1,30 @@
{
"wf" : {
"threshold" : "0.98",
"dedupRun" : "001",
"entityType" : "organization",
"orderField" : "legalname",
"queueMaxSize" : "2000",
"groupMaxSize" : "10",
"slidingWindowSize" : "200",
"rootBuilder" : [ "organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy" ],
"includeChildren" : "true"
},
"pace" : {
"clustering" : [
{ "name" : "ngrampairs", "fields" : [ "legalname" ], "params" : { "max" : "1", "ngramLen" : "3"} },
{ "name" : "suffixprefix", "fields" : [ "legalname" ], "params" : { "max" : "1", "len" : "3" } },
{ "name" : "spacetrimmingfieldvalue", "fields" : [ "legalshortname" ], "params" : { "randomLength" : "5" } }
],
"conditions" : [
{ "name" : "exactMatch", "fields" : [ "country" ] }
],
"model" : [
{ "name" : "legalname", "algo" : "JaroWinkler", "type" : "String", "weight" : "0.3", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value" },
{ "name" : "legalshortname", "algo" : "JaroWinkler", "type" : "String", "weight" : "0.3", "ignoreMissing" : "true", "path" : "organization/metadata/legalshortname/value" },
{ "name" : "country", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/country/classid" },
{ "name" : "websiteurl", "algo" : "urlMatcher", "type" : "URL", "weight" : "0.4", "ignoreMissing" : "true", "path" : "organization/metadata/websiteurl/value", "params" : {"host":"1.0", "path":"0.0"} }
],
"blacklists" : { }
}
}

27
dnet-dedup-test/src/main/resources/eu/dnetlib/pace/organization.test.pace.conf Normal file
View File

@ -0,0 +1,27 @@
{
"wf" : {
"threshold" : "0.98",
"dedupRun" : "001",
"entityType" : "organization",
"orderField" : "legalname",
"queueMaxSize" : "2000",
"groupMaxSize" : "10",
"slidingWindowSize" : "200",
"rootBuilder" : [ "organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy" ],
"includeChildren" : "true"
},
"pace" : {
"clustering" : [
{ "name" : "ngrampairs", "fields" : [ "legalname" ], "params" : { "max" : "1", "ngramLen" : "3"} },
{ "name" : "suffixprefix", "fields" : [ "legalname" ], "params" : { "max" : "1", "len" : "3" } }
],
"conditions" : [
{ "name" : "exactMatch", "fields" : [ "country" ] }
],
"model" : [
{ "name" : "legalname", "algo" : "Levenstein", "type" : "String", "weight" : "1.0", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value" },
{ "name" : "country", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/country/classid" }
],
"blacklists" : { }
}
}

5
dnet-dedup-test/src/main/resources/eu/dnetlib/pace/orgs.json Normal file
View File

@ -0,0 +1,5 @@
{"dateoftransformation":"2018-09-17","originalId":["corda__h2020::999838074"],"collectedfrom":[{"value":"CORDA - COmmon Research DAta Warehouse - Horizon 2020","key":"10|openaire____::a55eb91348674d853191f4f4fd73d078"}],"organization":{"metadata":{"eclegalbody":{"value":"true"},"eclegalperson":{"value":"true"},"ecinternationalorganization":{"value":"false"},"legalshortname":{"value":"UNIVR"},"ecresearchorganization":{"value":"true"},"ecnonprofit":{"value":"true"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.univr.it"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"UNIVERSITA DEGLI STUDI DI VERONA"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"true"},"ecsmevalidated":{"value":"false"}}},"dateofcollection":"2018-03-12","type":20,"id":"20|corda__h2020::3898c35de19616484a0e901a92a709f5"}
{"dateoftransformation":"2018-09-13","originalId":["corda_______::999838074"],"collectedfrom":[{"value":"CORDA - COmmon Research DAta Warehouse","key":"10|openaire____::b30dac7baac631f3da7c2bb18dd9891f"}],"organization":{"metadata":{"eclegalbody":{"value":"true"},"eclegalperson":{"value":"true"},"ecinternationalorganization":{"value":"false"},"legalshortname":{"value":"UNIVR"},"ecresearchorganization":{"value":"true"},"ecnonprofit":{"value":"true"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.univr.it"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"UNIVERSITA DEGLI STUDI DI VERONA"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"true"}}},"dateofcollection":"2018-03-12","type":20,"id":"20|corda_______::3898c35de19616484a0e901a92a709f5"}
{"dateoftransformation":"2018-09-17","originalId":["corda__h2020::999976687"],"collectedfrom":[{"value":"CORDA - COmmon Research DAta Warehouse - Horizon 2020","key":"10|openaire____::a55eb91348674d853191f4f4fd73d078"}],"organization":{"metadata":{"eclegalbody":{"value":"true"},"eclegalperson":{"value":"true"},"ecinternationalorganization":{"value":"false"},"legalshortname":{"value":"UNIGE"},"ecresearchorganization":{"value":"true"},"ecnonprofit":{"value":"true"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.unige.it"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"UNIVERSITA DEGLI STUDI DI GENOVA"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"true"},"ecsmevalidated":{"value":"false"}}},"dateofcollection":"2018-03-12","type":20,"id":"20|corda__h2020::67fba37704a39567853e54615c5371fe"}
{"dateoftransformation":"2018-09-13","originalId":["opendoar____::Università_degli_Studi_di_Genova"],"collectedfrom":[{"value":"OpenDOAR","key":"10|openaire____::47ce9e9f4fad46e732cff06419ecaabb"}],"organization":{"metadata":{"eclegalbody":{"value":"false"},"eclegalperson":{"value":"false"},"ecinternationalorganization":{"value":"false"},"ecresearchorganization":{"value":"false"},"ecnonprofit":{"value":"false"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"https://www.unige.it/"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"Università degli Studi di Genova"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"false"},"ecsmevalidated":{"value":"false"}}},"dateofcollection":"2016-09-01","type":20,"id":"20|opendoar____::fcd6c93c2863e6be9c6f6a66d761c92d"}
{"dateoftransformation":"2018-09-13","originalId":["opendoar____::Università_degli_Studi_di_Verona"],"collectedfrom":[{"value":"OpenDOAR","key":"10|openaire____::47ce9e9f4fad46e732cff06419ecaabb"}],"organization":{"metadata":{"eclegalbody":{"value":"false"},"eclegalperson":{"value":"false"},"ecinternationalorganization":{"value":"false"},"ecresearchorganization":{"value":"false"},"ecnonprofit":{"value":"false"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.univr.it/"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"Università degli Studi di Verona"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"false"},"ecsmevalidated":{"value":"false"}}},"dateofcollection":"2015-08-24","type":20,"id":"20|opendoar____::3f2f26e4bf71340e806ec956884fe34e"}

2
dnet-dedup-test/src/main/resources/eu/dnetlib/pace/orgs.test.json Normal file
View File

@ -0,0 +1,2 @@
{"dateoftransformation":"2018-09-17","originalId":["corda__h2020::999838074"],"collectedfrom":[{"value":"CORDA - COmmon Research DAta Warehouse - Horizon 2020","key":"10|openaire____::a55eb91348674d853191f4f4fd73d078"}],"organization":{"metadata":{"eclegalbody":{"value":"true"},"eclegalperson":{"value":"true"},"ecinternationalorganization":{"value":"false"},"legalshortname":{"value":"UNIVR"},"ecresearchorganization":{"value":"true"},"ecnonprofit":{"value":"true"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.univr.it"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"VERONA"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"true"},"ecsmevalidated":{"value":"false"}}},"dateofcollection":"2018-03-12","type":20,"id":"20|corda__h2020::3898c35de19616484a0e901a92a709f5"}
{"dateoftransformation":"2018-09-13","originalId":["corda_______::999838074"],"collectedfrom":[{"value":"CORDA - COmmon Research DAta Warehouse","key":"10|openaire____::b30dac7baac631f3da7c2bb18dd9891f"}],"organization":{"metadata":{"eclegalbody":{"value":"true"},"eclegalperson":{"value":"true"},"ecinternationalorganization":{"value":"false"},"legalshortname":{"value":"UNIVR"},"ecresearchorganization":{"value":"true"},"ecnonprofit":{"value":"true"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.univr.it"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"GENOVA"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"true"}}},"dateofcollection":"2018-03-12","type":20,"id":"20|corda_______::3898c35de19616484a0e901a92a709f5"}

2
dnet-dedup-test/src/main/resources/eu/dnetlib/pace/result.simple.pace.conf
View File

@ -1,7 +1,7 @@
{ {
"wf" : { "wf" : {
"threshold" : "0.99", "threshold" : "0.99",
"run" : "001", "dedupRun" : "001",
"entityType" : "result", "entityType" : "result",
"orderField" : "title", "orderField" : "title",
"queueMaxSize" : "2000", "queueMaxSize" : "2000",

420
dnet-dedup-test/src/test/java/eu/dnetlib/pace/AbstractProtoPaceTest.java
View File

@ -1,208 +1,212 @@
//package eu.dnetlib.pace; package eu.dnetlib.pace;
//
//import com.google.common.collect.Lists; import com.google.common.collect.Lists;
//import com.google.common.collect.Sets; import com.google.gson.Gson;
//import com.google.gson.Gson; import eu.dnetlib.data.proto.FieldTypeProtos.Author;
//import eu.dnetlib.data.proto.FieldTypeProtos.Author; import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;
//import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier; import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;
//import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty; import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty.Builder;
//import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty.Builder; import eu.dnetlib.data.proto.OafProtos.Oaf;
//import eu.dnetlib.data.proto.OafProtos.Oaf; import eu.dnetlib.data.proto.OafProtos.OafEntity;
//import eu.dnetlib.data.proto.OafProtos.OafEntity; import eu.dnetlib.data.proto.OrganizationProtos.Organization;
//import eu.dnetlib.data.proto.OrganizationProtos.Organization; import eu.dnetlib.data.proto.ResultProtos.Result;
//import eu.dnetlib.data.proto.ResultProtos.Result; import eu.dnetlib.pace.config.Config;
//import eu.dnetlib.pace.config.Config; import eu.dnetlib.pace.config.DedupConfig;
//import eu.dnetlib.pace.config.DedupConfig; import eu.dnetlib.pace.config.Type;
//import eu.dnetlib.pace.config.Type; import eu.dnetlib.pace.model.Field;
//import eu.dnetlib.pace.model.Field; import eu.dnetlib.pace.model.FieldValueImpl;
//import eu.dnetlib.pace.model.FieldValueImpl; import eu.dnetlib.pace.model.MapDocument;
//import eu.dnetlib.pace.model.MapDocument; import eu.dnetlib.pace.model.ProtoDocumentBuilder;
//import eu.dnetlib.pace.model.ProtoDocumentBuilder; import eu.dnetlib.pace.model.gt.GTAuthor;
//import eu.dnetlib.pace.model.gt.GTAuthor; import org.apache.commons.io.IOUtils;
//import org.apache.commons.io.IOUtils; import org.apache.commons.lang.RandomStringUtils;
//import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.StringUtils;
//import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.RandomUtils;
//import org.apache.commons.lang3.RandomUtils;
// import java.io.IOException;
//import java.io.IOException; import java.io.StringWriter;
//import java.io.StringWriter; import java.util.ArrayList;
//import java.util.ArrayList; import java.util.LinkedList;
//import java.util.LinkedList; import java.util.List;
//import java.util.List; import java.util.stream.Collectors;
//import java.util.Set; import java.util.stream.IntStream;
//import java.util.stream.Collectors;
//import java.util.stream.IntStream; public abstract class AbstractProtoPaceTest extends OafTest {
//
//public abstract class AbstractProtoPaceTest extends OafTest { protected DedupConfig getResultFullConf() {
// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.full.pace.conf"));
// protected DedupConfig getResultFullConf() { }
// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.full.pace.conf"));
// } protected DedupConfig getResultSimpleConf() {
// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.simple.pace.conf"));
// protected DedupConfig getResultSimpleConf() { }
// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.simple.pace.conf"));
// } protected DedupConfig getResultConf() {
// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.pace.conf"));
// protected DedupConfig getResultConf() { }
// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.pace.conf"));
// } protected DedupConfig getOrganizationSimpleConf() {
// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf"));
// protected DedupConfig getOrganizationSimpleConf() { }
// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf"));
// }
// protected DedupConfig getOrganizationTestConf() {
// protected DedupConfig getResultAuthorsConf() { return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.test.conf"));
// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.authors.pace.conf")); }
// }
//
// protected DedupConfig getResultProdConf() { protected DedupConfig getResultAuthorsConf() {
// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.prod.pace.conf")); return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.authors.pace.conf"));
// } }
//
// protected MapDocument author(final Config conf, final String id, final Oaf oaf) { protected DedupConfig getResultProdConf() {
// return ProtoDocumentBuilder.newInstance(id, oaf.getEntity(), conf.model()); return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.prod.pace.conf"));
// } }
//
// protected GTAuthor getGTAuthor(final String path) { protected MapDocument author(final Config conf, final String id, final Oaf oaf) {
// return ProtoDocumentBuilder.newInstance(id, oaf.getEntity(), conf.model());
// final Gson gson = new Gson(); }
//
// final String json = readFromClasspath(path); protected GTAuthor getGTAuthor(final String path) {
//
// final GTAuthor gta = gson.fromJson(json, GTAuthor.class); final Gson gson = new Gson();
//
// return gta; final String json = readFromClasspath(path);
// }
// final GTAuthor gta = gson.fromJson(json, GTAuthor.class);
// protected String readFromClasspath(final String filename) {
// final StringWriter sw = new StringWriter(); return gta;
// try { }
// IOUtils.copy(getClass().getResourceAsStream(filename), sw);
// return sw.toString(); protected String readFromClasspath(final String filename) {
// } catch (final IOException e) { final StringWriter sw = new StringWriter();
// throw new RuntimeException("cannot load resource from classpath: " + filename); try {
// } IOUtils.copy(getClass().getResourceAsStream(filename), sw);
// } return sw.toString();
// } catch (final IOException e) {
// protected MapDocument result(final Config config, final String id, final String title) { throw new RuntimeException("cannot load resource from classpath: " + filename);
// return result(config, id, title, null, new ArrayList<>(), null); }
// } }
//
// protected MapDocument result(final Config config, final String id, final String title, final String date) { protected MapDocument result(final Config config, final String id, final String title) {
// return result(config, id, title, date, new ArrayList<>(), null); return result(config, id, title, null, new ArrayList<>(), null);
// } }
//
// protected MapDocument result(final Config config, final String id, final String title, final String date, final List<String> pid) { protected MapDocument result(final Config config, final String id, final String title, final String date) {
// return result(config, id, title, date, pid, null); return result(config, id, title, date, new ArrayList<>(), null);
// } }
//
// protected MapDocument result(final Config config, final String id, final String title, final String date, final String pid) { protected MapDocument result(final Config config, final String id, final String title, final String date, final List<String> pid) {
// return result(config, id, title, date, pid, null); return result(config, id, title, date, pid, null);
// } }
//
// protected MapDocument result(final Config config, final String id, final String title, final String date, final String pid, final List<String> authors) { protected MapDocument result(final Config config, final String id, final String title, final String date, final String pid) {
// return result(config, id, title, date, Lists.newArrayList(pid), authors); return result(config, id, title, date, pid, null);
// } }
//
// static List<String> pidTypes = Lists.newArrayList(); protected MapDocument result(final Config config, final String id, final String title, final String date, final String pid, final List<String> authors) {
// static { return result(config, id, title, date, Lists.newArrayList(pid), authors);
// pidTypes.add("doi"); }
// //pidTypes.add("oai");
// //pidTypes.add("pmid"); static List<String> pidTypes = Lists.newArrayList();
// } static {
// pidTypes.add("doi");
// protected MapDocument result(final Config config, final String id, final String title, final String date, final List<String> pid, final List<String> authors) { //pidTypes.add("oai");
// final Result.Metadata.Builder metadata = Result.Metadata.newBuilder(); //pidTypes.add("pmid");
// if (!StringUtils.isBlank(title)) { }
// metadata.addTitle(getStruct(title, getQualifier("main title", "dnet:titles")));
// metadata.addTitle(getStruct(RandomStringUtils.randomAlphabetic(10), getQualifier("alternative title", "dnet:titles"))); protected MapDocument result(final Config config, final String id, final String title, final String date, final List<String> pid, final List<String> authors) {
// } final Result.Metadata.Builder metadata = Result.Metadata.newBuilder();
// if (!StringUtils.isBlank(date)) { if (!StringUtils.isBlank(title)) {
// metadata.setDateofacceptance(sf(date)); metadata.addTitle(getStruct(title, getQualifier("main title", "dnet:titles")));
// } metadata.addTitle(getStruct(RandomStringUtils.randomAlphabetic(10), getQualifier("alternative title", "dnet:titles")));
// }
// final OafEntity.Builder entity = oafEntity(id, eu.dnetlib.data.proto.TypeProtos.Type.result); if (!StringUtils.isBlank(date)) {
// final Result.Builder result = Result.newBuilder().setMetadata(metadata); metadata.setDateofacceptance(sf(date));
// }
// if (authors != null) {
// result.getMetadataBuilder().addAllAuthor( final OafEntity.Builder entity = oafEntity(id, eu.dnetlib.data.proto.TypeProtos.Type.result);
// IntStream.range(0, authors.size()) final Result.Builder result = Result.newBuilder().setMetadata(metadata);
// .mapToObj(i -> author(authors.get(i), i))
// .collect(Collectors.toCollection(LinkedList::new))); if (authors != null) {
// } result.getMetadataBuilder().addAllAuthor(
// IntStream.range(0, authors.size())
// entity.setResult(result); .mapToObj(i -> author(authors.get(i), i))
// .collect(Collectors.toCollection(LinkedList::new)));
// if (pid != null) { }
// for(String p : pid) {
// if (!StringUtils.isBlank(p)) { entity.setResult(result);
// entity.addPid(sp(p, pidTypes.get(RandomUtils.nextInt(0, pidTypes.size() - 1))));
// //entity.addPid(sp(RandomStringUtils.randomAlphabetic(10), "oai")); if (pid != null) {
// } for(String p : pid) {
// } if (!StringUtils.isBlank(p)) {
// } entity.addPid(sp(p, pidTypes.get(RandomUtils.nextInt(0, pidTypes.size() - 1))));
// //entity.addPid(sp(RandomStringUtils.randomAlphabetic(10), "oai"));
// final OafEntity build = entity.build(); }
// return ProtoDocumentBuilder.newInstance(id, build, config.model()); }
// } }
//
// private Author author(final String s, int rank) { final OafEntity build = entity.build();
// final eu.dnetlib.pace.model.Person p = new eu.dnetlib.pace.model.Person(s, false); return ProtoDocumentBuilder.newInstance(id, build, config.model());
// final Author.Builder author = Author.newBuilder(); }
// if (p.isAccurate()) {
// author.setName(p.getNormalisedFirstName()); private Author author(final String s, int rank) {
// author.setSurname(p.getNormalisedSurname()); final eu.dnetlib.pace.model.Person p = new eu.dnetlib.pace.model.Person(s, false);
// } final Author.Builder author = Author.newBuilder();
// author.setFullname(p.getNormalisedFullname()); if (p.isAccurate()) {
// author.setRank(rank); author.setName(p.getNormalisedFirstName());
// author.setSurname(p.getNormalisedSurname());
// return author.build(); }
// } author.setFullname(p.getNormalisedFullname());
// author.setRank(rank);
// private OafEntity.Builder oafEntity(final String id, final eu.dnetlib.data.proto.TypeProtos.Type type) {
// final OafEntity.Builder entity = OafEntity.newBuilder().setId(id).setType(type); return author.build();
// return entity; }
// }
// private OafEntity.Builder oafEntity(final String id, final eu.dnetlib.data.proto.TypeProtos.Type type) {
// protected MapDocument organization(final Config config, final String id, final String legalName) { final OafEntity.Builder entity = OafEntity.newBuilder().setId(id).setType(type);
// return organization(config, id, legalName, null); return entity;
// } }
//
// protected MapDocument organization(final Config config, final String id, final String legalName, final String legalShortName) { protected MapDocument organization(final Config config, final String id, final String legalName) {
// final Organization.Metadata.Builder metadata = Organization.Metadata.newBuilder(); return organization(config, id, legalName, null);
// if (legalName != null) { }
// metadata.setLegalname(sf(legalName));
// } protected MapDocument organization(final Config config, final String id, final String legalName, final String legalShortName) {
// if (legalShortName != null) { final Organization.Metadata.Builder metadata = Organization.Metadata.newBuilder();
// metadata.setLegalshortname(sf(legalShortName)); if (legalName != null) {
// } metadata.setLegalname(sf(legalName));
// }
// final OafEntity.Builder entity = oafEntity(id, eu.dnetlib.data.proto.TypeProtos.Type.result); if (legalShortName != null) {
// entity.setOrganization(Organization.newBuilder().setMetadata(metadata)); metadata.setLegalshortname(sf(legalShortName));
// }
// return ProtoDocumentBuilder.newInstance(id, entity.build(), config.model());
// } final OafEntity.Builder entity = oafEntity(id, eu.dnetlib.data.proto.TypeProtos.Type.result);
// entity.setOrganization(Organization.newBuilder().setMetadata(metadata));
// private StructuredProperty sp(final String pid, final String type) {
// final Builder pidSp = return ProtoDocumentBuilder.newInstance(id, entity.build(), config.model());
// StructuredProperty.newBuilder().setValue(pid) }
// .setQualifier(Qualifier.newBuilder().setClassid(type).setClassname(type).setSchemeid("dnet:pid_types").setSchemename("dnet:pid_types"));
// return pidSp.build(); private StructuredProperty sp(final String pid, final String type) {
// } final Builder pidSp =
// StructuredProperty.newBuilder().setValue(pid)
// protected Field title(final String s) { .setQualifier(Qualifier.newBuilder().setClassid(type).setClassname(type).setSchemeid("dnet:pid_types").setSchemename("dnet:pid_types"));
// return new FieldValueImpl(Type.String, "title", s); return pidSp.build();
// } }
//
// protected static Builder getStruct(final String value, final Qualifier.Builder qualifier) { protected Field title(final String s) {
// return StructuredProperty.newBuilder().setValue(value).setQualifier(qualifier); return new FieldValueImpl(Type.String, "title", s);
// } }
//
// /* protected static Builder getStruct(final String value, final Qualifier.Builder qualifier) {
// * protected static StringField.Builder sf(final String s) { return StringField.newBuilder().setValue(s); } return StructuredProperty.newBuilder().setValue(value).setQualifier(qualifier);
// * }
// * protected static Qualifier.Builder getQualifier(final String classname, final String schemename) { return
// * Qualifier.newBuilder().setClassid(classname).setClassname(classname).setSchemeid(schemename).setSchemename(schemename); } /*
// */ * protected static StringField.Builder sf(final String s) { return StringField.newBuilder().setValue(s); }
// *
//} * protected static Qualifier.Builder getQualifier(final String classname, final String schemename) { return
* Qualifier.newBuilder().setClassid(classname).setClassname(classname).setSchemeid(schemename).setSchemename(schemename); }
*/
}

892
dnet-dedup-test/src/test/java/eu/dnetlib/pace/OafTest.java
View File

File diff suppressed because one or more lines are too long

810
dnet-dedup-test/src/test/java/eu/dnetlib/pace/distance/DetectorTest.java
View File

@ -1,450 +1,360 @@
//package eu.dnetlib.pace.distance; package eu.dnetlib.pace.distance;
//
//import com.google.common.collect.Lists; import com.google.common.collect.Lists;
//import com.google.common.collect.Maps; import com.google.common.collect.Sets;
//import com.google.common.collect.Sets; import com.googlecode.protobuf.format.JsonFormat;
//import com.googlecode.protobuf.format.JsonFormat; import eu.dnetlib.data.proto.OafProtos;
//import eu.dnetlib.data.proto.OafProtos; import eu.dnetlib.pace.AbstractProtoPaceTest;
//import eu.dnetlib.pace.AbstractProtoPaceTest; import eu.dnetlib.pace.clustering.BlacklistAwareClusteringCombiner;
//import eu.dnetlib.pace.clustering.BlacklistAwareClusteringCombiner; import eu.dnetlib.pace.config.Config;
//import eu.dnetlib.pace.config.Config; import eu.dnetlib.pace.config.DedupConfig;
//import eu.dnetlib.pace.config.DedupConfig; import eu.dnetlib.pace.distance.eval.ScoreResult;
//import eu.dnetlib.pace.distance.eval.ScoreResult; import eu.dnetlib.pace.model.MapDocument;
//import eu.dnetlib.pace.model.MapDocument; import eu.dnetlib.pace.model.ProtoDocumentBuilder;
//import eu.dnetlib.pace.model.ProtoDocumentBuilder; import org.apache.commons.logging.Log;
//import org.apache.commons.io.IOUtils; import org.apache.commons.logging.LogFactory;
//import org.apache.commons.logging.Log; import org.junit.Ignore;
//import org.apache.commons.logging.LogFactory; import org.junit.Test;
//import org.junit.Ignore;
//import org.junit.Test; import java.io.IOException;
// import java.util.LinkedList;
//import java.io.IOException; import java.util.List;
//import java.util.List; import java.util.Set;
//import java.util.Map;
//import java.util.Set; import static org.junit.Assert.assertFalse;
//import java.util.stream.Collectors; import static org.junit.Assert.assertTrue;
//
//import static org.junit.Assert.assertFalse; public class DetectorTest extends AbstractProtoPaceTest {
//import static org.junit.Assert.assertTrue;
// private static final Log log = LogFactory.getLog(DetectorTest.class);
//public class DetectorTest extends AbstractProtoPaceTest {
// @Test
// private static final Log log = LogFactory.getLog(DetectorTest.class); public void testDistanceResultSimple() {
// final Config config = getResultSimpleConf();
// @Test final MapDocument resA = result(config, "A", "Recent results from CDF");
// public void testDistanceResultSimple() { final MapDocument resB = result(config, "B", "Recent results from CDF");
// final Config config = getResultSimpleConf(); final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final double d = sr.getScore();
// final MapDocument resA = result(config, "A", "Recent results from CDF"); log.info(String.format(" d ---> %s", d));
// final MapDocument resB = result(config, "B", "Recent results from CDF"); assertTrue(d == 1.0);
// }
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final double d = sr.getScore(); @Test
// log.info(String.format(" d ---> %s", d)); public void testDistanceResultSimpleMissingDates() {
// final Config config = getResultSimpleConf();
// assertTrue(d == 1.0); final MapDocument resA = result(config, "A", "Recent results from BES");
// } final MapDocument resB = result(config, "A", "Recent results from CES");
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// @Test final double d = sr.getScore();
// public void testDistanceResultSimpleMissingDates() { log.info(String.format(" d ---> %s", d));
// final Config config = getResultSimpleConf(); assertTrue(d > 0.97);
// }
// final MapDocument resA = result(config, "A", "Recent results from BES");
// final MapDocument resB = result(config, "A", "Recent results from CES"); @Test
// public void testDistanceResultInvalidDate() {
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); final Config config = getResultConf();
// final double d = sr.getScore(); final MapDocument resA = result(config, "A", "title title title 6BESR", "2013-01-05");
// log.info(String.format(" d ---> %s", d)); final MapDocument resB = result(config, "B", "title title title 6BESR", "qwerty");
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// assertTrue(d > 0.97); final double d = sr.getScore();
// } log.info(String.format(" d ---> %s", d));
// assertTrue(d == 1.0);
// @Test }
// public void testDistanceResultInvalidDate() {
// final Config config = getResultConf(); @Ignore
// @Test
// final MapDocument resA = result(config, "A", "title title title 6BESR", "2013-01-05"); public void testDistanceResultMissingOneDate() {
// final MapDocument resB = result(config, "B", "title title title 6BESR", "qwerty"); final Config config = getResultConf();
// final MapDocument resA = result(config, "A", "title title title 6BESR", null);
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); final MapDocument resB = result(config, "B", "title title title 6CLER", "2012-02");
// final double d = sr.getScore(); final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// log.info(String.format(" d ---> %s", d)); double d = sr.getScore();
// log.info(String.format(" d ---> %s", d));
// assertTrue(d == 1.0); assertTrue((d > 0.9) && (d < 1.0));
// } }
//
// @Ignore @Ignore
// @Test @Test
// public void testDistanceResultMissingOneDate() { public void testDistanceResult() {
// final Config config = getResultConf(); final Config config = getResultConf();
// final MapDocument resA = result(config, "A", "title title title BES", "");
// final MapDocument resA = result(config, "A", "title title title 6BESR", null); final MapDocument resB = result(config, "B", "title title title CLEO");
// final MapDocument resB = result(config, "B", "title title title 6CLER", "2012-02"); final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// double d = sr.getScore();
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); log.info(String.format(" d ---> %s", d));
// double d = sr.getScore(); assertTrue((d > 0.9) && (d < 1.0));
// log.info(String.format(" d ---> %s", d)); }
//
// assertTrue((d > 0.9) && (d < 1.0)); @Ignore
// } @Test
// public void testDistanceResultMissingTwoDate() {
// @Ignore final Config config = getResultConf();
// @Test final MapDocument resA = result(config, "A", "bellaciao");
// public void testDistanceResult() { final MapDocument resB = result(config, "B", "bellocioa");
// final Config config = getResultConf(); final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// double d = sr.getScore();
// final MapDocument resA = result(config, "A", "title title title BES", ""); log.info(String.format(" d ---> %s", d));
// final MapDocument resB = result(config, "B", "title title title CLEO"); assertTrue((d > 0.9) && (d < 1.0));
// }
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// double d = sr.getScore(); @Ignore
// log.info(String.format(" d ---> %s", d)); @Test
// public void testDistanceOrganizationIgnoreMissing() {
// assertTrue((d > 0.9) && (d < 1.0)); final Config config = getOrganizationSimpleConf();
// } final MapDocument orgA = organization(config, "A", "CONSIGLIO NAZIONALE DELLE RICERCHE");
// final MapDocument orgB = organization(config, "B", "CONSIGLIO NAZIONALE DELLE RICERCHE", "CNR");
// @Ignore final ScoreResult sr = new PaceDocumentDistance().between(orgA, orgB, config);
// @Test final double d = sr.getScore();
// public void testDistanceResultMissingTwoDate() { log.info(String.format(" d ---> %s", d));
// final Config config = getResultConf(); assertTrue(d > 0.99);
// }
// final MapDocument resA = result(config, "A", "bellaciao");
// final MapDocument resB = result(config, "B", "bellocioa");
// @Test
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); public void testDistanceOrganizations() {
// double d = sr.getScore(); final Config config = getOrganizationTestConf();
// log.info(String.format(" d ---> %s", d)); final MapDocument orgA = organization(config, "A", "UNIVERSITA DEGLI STUDI DI VERONA");
// final MapDocument orgB = organization(config, "B", "UNIVERSITY OF GENOVA");
// assertTrue((d > 0.9) && (d < 1.0)); final ScoreResult sr = new PaceDocumentDistance().between(orgA, orgB, config);
// } final double d = sr.getScore();
// log.info(String.format(" d ---> %s", d));
// @Ignore }
// @Test
// public void testDistanceOrganizationIgnoreMissing() {
// @Test
// final Config config = getOrganizationSimpleConf(); public void testDistanceResultCase1() {
// final Config config = getResultConf();
// final MapDocument orgA = organization(config, "A", "CONSIGLIO NAZIONALE DELLE RICERCHE"); final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003");
// final MapDocument orgB = organization(config, "B", "CONSIGLIO NAZIONALE DELLE RICERCHE", "CNR"); final MapDocument resB = result(config, "B", "Search for the Standard Model Higgs Boson", "2003");
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final ScoreResult sr = new PaceDocumentDistance().between(orgA, orgB, config); double d = sr.getScore();
// final double d = sr.getScore(); log.info(String.format(" d ---> %s", d));
// log.info(String.format(" d ---> %s", d)); assertTrue((d > 0.9) && (d < 1.0));
// }
// assertTrue(d > 0.99);
// } @Test
// public void testDistanceResultCaseDoiMatch1() {
// @Test final Config config = getResultConf();
// public void testDistanceResultCase1() { final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003", "10.1594/PANGAEA.726855");
// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", "10.1594/PANGAEA.726855");
// final Config config = getResultConf(); final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// double d = sr.getScore();
// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003"); log.info(String.format(" d ---> %s", d));
// final MapDocument resB = result(config, "B", "Search for the Standard Model Higgs Boson", "2003"); assertTrue("exact DOIs will produce an exact match", d == 1.0);
// }
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// double d = sr.getScore(); @Test
// log.info(String.format(" d ---> %s", d)); public void testDistanceResultCaseDoiMatch2() {
// final Config config = getResultConf();
// assertTrue((d > 0.9) && (d < 1.0)); final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1594/PANGAEA.726855");
// } final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2005", "10.1594/PANGAEA.726855");
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// @Test double d = sr.getScore();
// public void testDistanceResultCaseDoiMatch1() { log.info(String.format(" d ---> %s", d));
// final Config config = getResultConf(); assertTrue("exact DOIs will produce an exact match, regardless of different titles or publication years", d == 1.0);
// }
// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003", "10.1594/PANGAEA.726855");
// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", "10.1594/PANGAEA.726855"); @Test
// public void testDistanceResultCaseDoiMatch3() {
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); final Config config = getResultConf();
// double d = sr.getScore(); final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024");
// log.info(String.format(" d ---> %s", d)); final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003");
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// assertTrue("exact DOIs will produce an exact match", d == 1.0); double d = sr.getScore();
// } log.info(String.format(" d ---> %s", d));
// assertTrue("a missing DOI will casue the comparsion to continue with the following conditions", d == 1.0);
// @Test }
// public void testDistanceResultCaseDoiMatch2() {
// final Config config = getResultConf(); @Test
// public void testDistanceResultCaseDoiMatch4() {
// final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1594/PANGAEA.726855"); final Config config = getResultConf();
// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2005", "10.1594/PANGAEA.726855"); final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024");
// final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2005");
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// double d = sr.getScore(); double d = sr.getScore();
// log.info(String.format(" d ---> %s", d)); log.info(String.format(" d ---> %s", d));
// assertTrue("a missing DOI, comparsion continues with the following conditions, different publication years will drop the score to 0", d == 0.0);
// assertTrue("exact DOIs will produce an exact match, regardless of different titles or publication years", d == 1.0); }
// }
// @Test
// @Test public void testDistanceResultCaseDoiMatch5() {
// public void testDistanceResultCaseDoiMatch3() { final Config config = getResultConf();
// final Config config = getResultConf(); final MapDocument resA = result(config, "A", "Search for the Standard Model Higgs Boson", "2003", "10.1016/j.jmb.2010.12.020");
// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003");
// final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024"); final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003"); double d = sr.getScore();
// log.info(String.format(" d ---> %s", d));
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); assertTrue("a missing DOI, comparsion continues with the following conditions", (d > 0.9) && (d < 1.0));
// double d = sr.getScore(); }
// log.info(String.format(" d ---> %s", d));
// @Test
// assertTrue("a missing DOI will casue the comparsion to continue with the following conditions", d == 1.0); public void testDistanceResultCaseDoiMatch6() {
// } final Config config = getResultConf();
// final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024");
// @Test final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003", "anotherDifferentDOI");
// public void testDistanceResultCaseDoiMatch4() { final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final Config config = getResultConf(); double d = sr.getScore();
// log.info(String.format(" d ---> %s", d));
// final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024"); assertTrue("different DOIs will NOT drop the score to 0, then evaluate other fields", d == 1.0);
// final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2005"); }
//
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); @Test
// double d = sr.getScore(); public void testDistanceResultCaseDoiMatch7() {
// log.info(String.format(" d ---> %s", d)); final Config config = getResultConf();
// final MapDocument resA = result(config, "A", "Adrenal Insufficiency asd asd", "1951", Lists.newArrayList("PMC2037944", "axdsds"));
// assertTrue("a missing DOI, comparsion continues with the following conditions, different publication years will drop the score to 0", d == 0.0); final MapDocument resB = result(config, "B", "Adrenal Insufficiency", "1951", "PMC2037944");
// } final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// double d = sr.getScore();
// @Test log.info(String.format(" d ---> %s", d));
// public void testDistanceResultCaseDoiMatch5() { assertTrue("different DOIs will drop the score to 0, regardless of the other fields", d > 0.9 & d < 1);
// }
// final Config config = getResultConf();
// // http://dx.doi.org/10.1594/PANGAEA.726855 doi:10.1594/PANGAEA.726855
// final MapDocument resA = result(config, "A", "Search for the Standard Model Higgs Boson", "2003", "10.1016/j.jmb.2010.12.020"); @Test
// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003"); public void testDistanceResultCaseAuthor1() {
// final Config config = getResultAuthorsConf();
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); final List<String> authorsA = Lists.newArrayList("a", "b", "c", "d");
// double d = sr.getScore(); final List<String> authorsB = Lists.newArrayList("a", "b", "c");
// log.info(String.format(" d ---> %s", d)); final List<String> pid = Lists.newArrayList();
// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA);
// assertTrue("a missing DOI, comparsion continues with the following conditions", (d > 0.9) && (d < 1.0)); final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB);
// } final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final double d = sr.getScore();
// @Test log.info(String.format(" d ---> %s", d));
// public void testDistanceResultCaseDoiMatch6() { assertTrue(d == 0.0);
// final Config config = getResultConf(); }
//
// final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024"); @Test
// final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003", "anotherDifferentDOI"); public void testDistanceResultCaseAuthor2() {
// final Config config = getResultAuthorsConf();
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); final List<String> authorsA = Lists.newArrayList("a", "b", "c");
// double d = sr.getScore(); final List<String> authorsB = Lists.newArrayList("a", "b", "c");
// log.info(String.format(" d ---> %s", d)); final List<String> pid = Lists.newArrayList();
// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA);
// assertTrue("different DOIs will NOT drop the score to 0, then evaluate other fields", d == 1.0); final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB);
// } final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final double d = sr.getScore();
// @Test log.info(String.format(" d ---> %s", d));
// public void testDistanceResultCaseDoiMatch7() { assertTrue(d == 1.0);
// final Config config = getResultConf(); }
//
// final MapDocument resA = result(config, "A", "Adrenal Insufficiency asd asd", "1951", Lists.newArrayList("PMC2037944", "axdsds")); @Test
// final MapDocument resB = result(config, "B", "Adrenal Insufficiency", "1951", "PMC2037944"); public void testDistanceResultCaseAuthor3() {
// final Config config = getResultAuthorsConf();
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); final List<String> authorsA = Lists.newArrayList("Bardi, A.", "Manghi, P.", "Artini, M.");
// double d = sr.getScore(); final List<String> authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele");
// log.info(String.format(" d ---> %s", d)); final List<String> pid = Lists.newArrayList();
// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA);
// assertTrue("different DOIs will drop the score to 0, regardless of the other fields", d > 0.9 & d < 1); final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB);
// } final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// double d = sr.getScore();
// // http://dx.doi.org/10.1594/PANGAEA.726855 doi:10.1594/PANGAEA.726855 log.info(String.format(" d ---> %s", d));
// assertTrue((d > 0.9) && (d < 1.0));
// @Test }
// public void testDistanceResultCaseAuthor1() {
// @Test
// final Config config = getResultAuthorsConf(); public void testDistanceResultCaseAuthor4() {
// final Config config = getResultAuthorsConf();
// final List<String> authorsA = Lists.newArrayList("a", "b", "c", "d"); final List<String> authorsA = Lists.newArrayList("Bardi, Alessia", "Manghi, Paolo", "Artini, Michele", "a");
// final List<String> authorsB = Lists.newArrayList("a", "b", "c"); final List<String> authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele");
// final List<String> pid = Lists.newArrayList(); final List<String> pid = Lists.newArrayList();
// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA);
// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB);
// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final double d = sr.getScore();
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); log.info(String.format(" d ---> %s", d));
// final double d = sr.getScore(); // assertTrue(d.getScore() == 0.0);
// log.info(String.format(" d ---> %s", d)); }
//
// assertTrue(d == 0.0); @Test
// } public void testDistanceResultNoPidsConf() {
// final Config config = getResultFullConf();
// @Test final MapDocument resA =
// public void testDistanceResultCaseAuthor2() { result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010");
// final MapDocument resB =
// final Config config = getResultAuthorsConf(); result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reportsX", "2010");
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final List<String> authorsA = Lists.newArrayList("a", "b", "c"); final double s = sr.getScore();
// final List<String> authorsB = Lists.newArrayList("a", "b", "c"); log.info(sr.toString());
// final List<String> pid = Lists.newArrayList(); log.info(String.format(" s ---> %s", s));
// // assertTrue(d.getScore() == 0.0);
// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); }
// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB);
// @Test
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); public void testDistanceResultPidsConf() {
// final double d = sr.getScore(); final Config config = getResultFullConf();
// log.info(String.format(" d ---> %s", d)); final List<String> authorsA = Lists.newArrayList("Nagarajan Pranesh", "Guy Vautier", "Punyanganie de Silva");
// final List<String> authorsB = Lists.newArrayList("Pranesh Nagarajan", "Vautier Guy", "de Silva Punyanganie");
// assertTrue(d == 1.0); final List<String> pidA = Lists.newArrayList("10.1186/1752-1947-4-299", "a", "b");
// } final MapDocument resA =
// result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010",
// @Test pidA, authorsA);
// public void testDistanceResultCaseAuthor3() { final List<String> pidB = Lists.newArrayList("c", "a", "10.1186/1752-1947-4-299", "d");
// final MapDocument resB =
// final Config config = getResultAuthorsConf(); result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reportsX", "2010",
// pidB, authorsB);
// final List<String> authorsA = Lists.newArrayList("Bardi, A.", "Manghi, P.", "Artini, M."); final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final List<String> authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele"); final double s = sr.getScore();
// final List<String> pid = Lists.newArrayList(); log.info(sr.toString());
// log.info(String.format(" s ---> %s", s));
// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); // assertTrue(d.getScore() == 0.0);
// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); }
//
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); @Test
// double d = sr.getScore(); public void testDistanceResultFullConf() {
// log.info(String.format(" d ---> %s", d)); final Config config = getResultFullConf();
// final List<String> authorsA = Lists.newArrayList("Nagarajan Pranesh", "Guy Vautier", "Punyanganie de Silva");
// assertTrue((d > 0.9) && (d < 1.0)); final List<String> authorsB = Lists.newArrayList("Pranesh Nagarajan", "Vautier Guy", "de Silva Punyanganie");
// } final MapDocument resA =
// result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010",
// @Test "10.1186/1752-1947-4-299", authorsA);
// public void testDistanceResultCaseAuthor4() { final MapDocument resB =
// result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010",
// final Config config = getResultAuthorsConf(); "10.1186/1752-1947-4-299", authorsB);
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final List<String> authorsA = Lists.newArrayList("Bardi, Alessia", "Manghi, Paolo", "Artini, Michele", "a"); final double d = sr.getScore();
// final List<String> authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele"); log.info(String.format(" d ---> %s", d));
// final List<String> pid = Lists.newArrayList(); // assertTrue(d.getScore() == 0.0);
// }
// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA);
// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); @Ignore
// @Test
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); public void testDistance() throws IOException {
// final double d = sr.getScore(); final DedupConfig conf = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.prod.pace.json"));
// log.info(String.format(" d ---> %s", d)); final MapDocument crossref = asMapDocument(conf, "/eu/dnetlib/pace/crossref.json");
// final MapDocument alicante = asMapDocument(conf, "/eu/dnetlib/pace/alicante.json");
// // assertTrue(d.getScore() == 0.0); final ScoreResult result = new PaceDocumentDistance().between(crossref, alicante, conf);
// } log.info("score = " + result);
// }
// @Test
// public void testDistanceResultNoPidsConf() { @Ignore
// @Test
// final Config config = getResultFullConf(); public void testDistanceOrgs() throws IOException {
// final DedupConfig conf = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf"));
// final MapDocument resA = final MapDocument orgA = asMapDocument(conf, readFromClasspath("/eu/dnetlib/pace/organization1.json"));
// result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010"); final MapDocument orgB = asMapDocument(conf, readFromClasspath("/eu/dnetlib/pace/organization2.json"));
// Set<String> keysA = getGroupingKeys(conf, orgA);
// final MapDocument resB = Set<String> keysB = getGroupingKeys(conf, orgB);
// result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reportsX", "2010"); assertFalse(String.format("A: %s\nB: %s", keysA, keysB), Sets.intersection(keysA, keysB).isEmpty());
// log.info("clustering keys A = " + getGroupingKeys(conf, orgA));
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); log.info("clustering keys B = " + getGroupingKeys(conf, orgB));
// final double s = sr.getScore(); final ScoreResult result = new PaceDocumentDistance().between(orgA, orgB, conf);
// log.info("score = " + result);
// log.info(sr.toString()); log.info("distance = " + result.getScore());
// log.info(String.format(" s ---> %s", s)); }
// // assertTrue(d.getScore() == 0.0);
// } private Set<String> getGroupingKeys(DedupConfig conf, MapDocument doc) {
// return Sets.newHashSet(BlacklistAwareClusteringCombiner.filterAndCombine(doc, conf));
// @Test }
// public void testDistanceResultPidsConf() {
// private MapDocument asMapDocument(DedupConfig conf, final String json) {
// final Config config = getResultFullConf(); OafProtos.OafEntity.Builder b = OafProtos.OafEntity.newBuilder();
// try {
// final List<String> authorsA = Lists.newArrayList("Nagarajan Pranesh", "Guy Vautier", "Punyanganie de Silva"); JsonFormat.merge(json, b);
// final List<String> authorsB = Lists.newArrayList("Pranesh Nagarajan", "Vautier Guy", "de Silva Punyanganie"); } catch (JsonFormat.ParseException e) {
// throw new IllegalArgumentException(e);
// final List<String> pidA = Lists.newArrayList("10.1186/1752-1947-4-299", "a", "b"); }
// final MapDocument resA = return ProtoDocumentBuilder.newInstance(b.getId(), b.build(), conf.getPace().getModel());
// result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010",
// pidA, authorsA); }
// }
// final List<String> pidB = Lists.newArrayList("c", "a", "10.1186/1752-1947-4-299", "d");
// final MapDocument resB =
// result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reportsX", "2010",
// pidB, authorsB);
//
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final double s = sr.getScore();
// log.info(sr.toString());
// log.info(String.format(" s ---> %s", s));
//
// // assertTrue(d.getScore() == 0.0);
// }
//
// @Test
// public void testDistanceResultFullConf() {
//
// final Config config = getResultFullConf();
//
// final List<String> authorsA = Lists.newArrayList("Nagarajan Pranesh", "Guy Vautier", "Punyanganie de Silva");
// final List<String> authorsB = Lists.newArrayList("Pranesh Nagarajan", "Vautier Guy", "de Silva Punyanganie");
//
// final MapDocument resA =
// result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010",
// "10.1186/1752-1947-4-299", authorsA);
//
// final MapDocument resB =
// result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010",
// "10.1186/1752-1947-4-299", authorsB);
//
// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);
// final double d = sr.getScore();
// log.info(String.format(" d ---> %s", d));
//
// // assertTrue(d.getScore() == 0.0);
// }
//
// @Ignore
// @Test
// public void testDistance() throws IOException {
//
// final DedupConfig conf = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.prod.pace.json"));
//
// final MapDocument crossref = asMapDocument(conf, "/eu/dnetlib/pace/crossref.json");
// final MapDocument alicante = asMapDocument(conf, "/eu/dnetlib/pace/alicante.json");
//
// final ScoreResult result = new PaceDocumentDistance().between(crossref, alicante, conf);
//
// log.info("score = " + result);
//
// }
//
// @Ignore
// @Test
// public void testDistanceOrgs() throws IOException {
//
// final DedupConfig conf = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf"));
//
// final MapDocument orgA = asMapDocument(conf, readFromClasspath("/eu/dnetlib/pace/organization1.json"));
// final MapDocument orgB = asMapDocument(conf, readFromClasspath("/eu/dnetlib/pace/organization2.json"));
//
// Set<String> keysA = getGroupingKeys(conf, orgA);
// Set<String> keysB = getGroupingKeys(conf, orgB);
//
// assertFalse(String.format("A: %s\nB: %s", keysA, keysB), Sets.intersection(keysA, keysB).isEmpty());
//
// log.info("clustering keys A = " + getGroupingKeys(conf, orgA));
// log.info("clustering keys B = " + getGroupingKeys(conf, orgB));
//
// final ScoreResult result = new PaceDocumentDistance().between(orgA, orgB, conf);
//
// log.info("score = " + result);
// log.info("distance = " + result.getScore());
// }
//
// private Set<String> getGroupingKeys(DedupConfig conf, MapDocument doc) {
// return Sets.newHashSet(BlacklistAwareClusteringCombiner.filterAndCombine(doc, conf));
// }
//
// private MapDocument asMapDocument(DedupConfig conf, final String json) {
// OafProtos.OafEntity.Builder b = OafProtos.OafEntity.newBuilder();
// try {
// JsonFormat.merge(json, b);
// } catch (JsonFormat.ParseException e) {
// throw new IllegalArgumentException(e);
// }
// return ProtoDocumentBuilder.newInstance(b.getId(), b.build(), conf.getPace().getModel());
// }
//
//
//}

2471
dnet-dedup-test/src/test/java/eu/dnetlib/pace/distance/DiffPatchMatch.java Normal file
View File

File diff suppressed because it is too large Load Diff

27
dnet-dedup-test/src/test/resources/eu/dnetlib/pace/organization.test.conf Normal file
View File

@ -0,0 +1,27 @@
{
"wf" : {
"threshold" : "0.98",
"dedupRun" : "001",
"entityType" : "organization",
"orderField" : "legalname",
"queueMaxSize" : "2000",
"groupMaxSize" : "10",
"slidingWindowSize" : "200",
"rootBuilder" : [ "organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy" ],
"includeChildren" : "true"
},
"pace" : {
"clustering" : [
{ "name" : "ngrampairs", "fields" : [ "legalname" ], "params" : { "max" : "1", "ngramLen" : "3"} },
{ "name" : "suffixprefix", "fields" : [ "legalname" ], "params" : { "max" : "1", "len" : "3" } }
],
"conditions" : [
{ "name" : "exactMatch", "fields" : [ "country" ] }
],
"model" : [
{ "name" : "legalname", "algo" : "JaroWinkler", "type" : "String", "weight" : "1.0", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value" },
{ "name" : "country", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/country/classid" }
],
"blacklists" : { }
}
}

2
dnet-dedup-test/src/test/resources/eu/dnetlib/pace/result.simple.pace.conf
View File

@ -1,7 +1,7 @@
{ {
"wf" : { "wf" : {
"threshold" : "0.99", "threshold" : "0.99",
"run" : "001", "dedupRun" : "001",
"entityType" : "result", "entityType" : "result",
"orderField" : "title", "orderField" : "title",
"queueMaxSize" : "2000", "queueMaxSize" : "2000",

2471
dnet-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java Normal file
View File

File diff suppressed because it is too large Load Diff