diff --git a/dnet-dedup-test/src/main/java/eu/dnetlib/SparkTest.java b/dnet-dedup-test/src/main/java/eu/dnetlib/SparkTest.java index b78ba82..e50cba0 100644 --- a/dnet-dedup-test/src/main/java/eu/dnetlib/SparkTest.java +++ b/dnet-dedup-test/src/main/java/eu/dnetlib/SparkTest.java @@ -22,22 +22,24 @@ import scala.Tuple2; import java.io.IOException; import java.io.StringWriter; +import java.net.URL; import java.util.Set; import java.util.stream.Collectors; - public class SparkTest { public static SparkCounter counter ; private static final Log log = LogFactory.getLog(SparkTest.class); public static void main(String[] args) { final JavaSparkContext context = new JavaSparkContext(new SparkConf().setAppName("Deduplication").setMaster("local[*]")); - final JavaRDD dataRDD = context.textFile("file:///Users/miconis/Downloads/dumps/organizations_sample.json"); + + final URL dataset = SparkTest.class.getResource("/eu/dnetlib/pace/orgs.test.json"); + final JavaRDD dataRDD = context.textFile(dataset.getPath()); counter = new SparkCounter(context); //read the configuration from the classpath - final DedupConfig config = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf")); + final DedupConfig config = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.test.pace.conf")); BlockProcessor.constructAccumulator(config); BlockProcessor.accumulators.forEach(acc -> { diff --git a/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/organization.beta.pace.conf b/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/organization.beta.pace.conf new file mode 100644 index 0000000..fc01d59 --- /dev/null +++ b/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/organization.beta.pace.conf @@ -0,0 +1,30 @@ +{ + "wf" : { + "threshold" : "0.98", + "dedupRun" : "001", + "entityType" : "organization", + "orderField" : "legalname", + "queueMaxSize" : "2000", + "groupMaxSize" : "10", + "slidingWindowSize" : "200", + "rootBuilder" : [ "organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy" ], + "includeChildren" : "true" + }, + "pace" : { + "clustering" : [ + { "name" : "ngrampairs", "fields" : [ "legalname" ], "params" : { "max" : "1", "ngramLen" : "3"} }, + { "name" : "suffixprefix", "fields" : [ "legalname" ], "params" : { "max" : "1", "len" : "3" } }, + { "name" : "spacetrimmingfieldvalue", "fields" : [ "legalshortname" ], "params" : { "randomLength" : "5" } } + ], + "conditions" : [ + { "name" : "exactMatch", "fields" : [ "country" ] } + ], + "model" : [ + { "name" : "legalname", "algo" : "JaroWinkler", "type" : "String", "weight" : "0.3", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value" }, + { "name" : "legalshortname", "algo" : "JaroWinkler", "type" : "String", "weight" : "0.3", "ignoreMissing" : "true", "path" : "organization/metadata/legalshortname/value" }, + { "name" : "country", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/country/classid" }, + { "name" : "websiteurl", "algo" : "urlMatcher", "type" : "URL", "weight" : "0.4", "ignoreMissing" : "true", "path" : "organization/metadata/websiteurl/value", "params" : {"host":"1.0", "path":"0.0"} } + ], + "blacklists" : { } + } +} \ No newline at end of file diff --git a/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/organization.test.pace.conf b/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/organization.test.pace.conf new file mode 100644 index 0000000..dabdbd3 --- /dev/null +++ b/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/organization.test.pace.conf @@ -0,0 +1,27 @@ +{ + "wf" : { + "threshold" : "0.98", + "dedupRun" : "001", + "entityType" : "organization", + "orderField" : "legalname", + "queueMaxSize" : "2000", + "groupMaxSize" : "10", + "slidingWindowSize" : "200", + "rootBuilder" : [ "organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy" ], + "includeChildren" : "true" + }, + "pace" : { + "clustering" : [ + { "name" : "ngrampairs", "fields" : [ "legalname" ], "params" : { "max" : "1", "ngramLen" : "3"} }, + { "name" : "suffixprefix", "fields" : [ "legalname" ], "params" : { "max" : "1", "len" : "3" } } + ], + "conditions" : [ + { "name" : "exactMatch", "fields" : [ "country" ] } + ], + "model" : [ + { "name" : "legalname", "algo" : "Levenstein", "type" : "String", "weight" : "1.0", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value" }, + { "name" : "country", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/country/classid" } + ], + "blacklists" : { } + } +} \ No newline at end of file diff --git a/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/orgs.json b/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/orgs.json new file mode 100644 index 0000000..593117e --- /dev/null +++ b/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/orgs.json @@ -0,0 +1,5 @@ +{"dateoftransformation":"2018-09-17","originalId":["corda__h2020::999838074"],"collectedfrom":[{"value":"CORDA - COmmon Research DAta Warehouse - Horizon 2020","key":"10|openaire____::a55eb91348674d853191f4f4fd73d078"}],"organization":{"metadata":{"eclegalbody":{"value":"true"},"eclegalperson":{"value":"true"},"ecinternationalorganization":{"value":"false"},"legalshortname":{"value":"UNIVR"},"ecresearchorganization":{"value":"true"},"ecnonprofit":{"value":"true"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.univr.it"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"UNIVERSITA DEGLI STUDI DI VERONA"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"true"},"ecsmevalidated":{"value":"false"}}},"dateofcollection":"2018-03-12","type":20,"id":"20|corda__h2020::3898c35de19616484a0e901a92a709f5"} +{"dateoftransformation":"2018-09-13","originalId":["corda_______::999838074"],"collectedfrom":[{"value":"CORDA - COmmon Research DAta Warehouse","key":"10|openaire____::b30dac7baac631f3da7c2bb18dd9891f"}],"organization":{"metadata":{"eclegalbody":{"value":"true"},"eclegalperson":{"value":"true"},"ecinternationalorganization":{"value":"false"},"legalshortname":{"value":"UNIVR"},"ecresearchorganization":{"value":"true"},"ecnonprofit":{"value":"true"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.univr.it"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"UNIVERSITA DEGLI STUDI DI VERONA"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"true"}}},"dateofcollection":"2018-03-12","type":20,"id":"20|corda_______::3898c35de19616484a0e901a92a709f5"} +{"dateoftransformation":"2018-09-17","originalId":["corda__h2020::999976687"],"collectedfrom":[{"value":"CORDA - COmmon Research DAta Warehouse - Horizon 2020","key":"10|openaire____::a55eb91348674d853191f4f4fd73d078"}],"organization":{"metadata":{"eclegalbody":{"value":"true"},"eclegalperson":{"value":"true"},"ecinternationalorganization":{"value":"false"},"legalshortname":{"value":"UNIGE"},"ecresearchorganization":{"value":"true"},"ecnonprofit":{"value":"true"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.unige.it"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"UNIVERSITA DEGLI STUDI DI GENOVA"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"true"},"ecsmevalidated":{"value":"false"}}},"dateofcollection":"2018-03-12","type":20,"id":"20|corda__h2020::67fba37704a39567853e54615c5371fe"} +{"dateoftransformation":"2018-09-13","originalId":["opendoar____::Università_degli_Studi_di_Genova"],"collectedfrom":[{"value":"OpenDOAR","key":"10|openaire____::47ce9e9f4fad46e732cff06419ecaabb"}],"organization":{"metadata":{"eclegalbody":{"value":"false"},"eclegalperson":{"value":"false"},"ecinternationalorganization":{"value":"false"},"ecresearchorganization":{"value":"false"},"ecnonprofit":{"value":"false"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"https://www.unige.it/"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"Università degli Studi di Genova"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"false"},"ecsmevalidated":{"value":"false"}}},"dateofcollection":"2016-09-01","type":20,"id":"20|opendoar____::fcd6c93c2863e6be9c6f6a66d761c92d"} +{"dateoftransformation":"2018-09-13","originalId":["opendoar____::Università_degli_Studi_di_Verona"],"collectedfrom":[{"value":"OpenDOAR","key":"10|openaire____::47ce9e9f4fad46e732cff06419ecaabb"}],"organization":{"metadata":{"eclegalbody":{"value":"false"},"eclegalperson":{"value":"false"},"ecinternationalorganization":{"value":"false"},"ecresearchorganization":{"value":"false"},"ecnonprofit":{"value":"false"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.univr.it/"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"Università degli Studi di Verona"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"false"},"ecsmevalidated":{"value":"false"}}},"dateofcollection":"2015-08-24","type":20,"id":"20|opendoar____::3f2f26e4bf71340e806ec956884fe34e"} \ No newline at end of file diff --git a/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/orgs.test.json b/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/orgs.test.json new file mode 100644 index 0000000..ef434c1 --- /dev/null +++ b/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/orgs.test.json @@ -0,0 +1,2 @@ +{"dateoftransformation":"2018-09-17","originalId":["corda__h2020::999838074"],"collectedfrom":[{"value":"CORDA - COmmon Research DAta Warehouse - Horizon 2020","key":"10|openaire____::a55eb91348674d853191f4f4fd73d078"}],"organization":{"metadata":{"eclegalbody":{"value":"true"},"eclegalperson":{"value":"true"},"ecinternationalorganization":{"value":"false"},"legalshortname":{"value":"UNIVR"},"ecresearchorganization":{"value":"true"},"ecnonprofit":{"value":"true"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.univr.it"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"VERONA"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"true"},"ecsmevalidated":{"value":"false"}}},"dateofcollection":"2018-03-12","type":20,"id":"20|corda__h2020::3898c35de19616484a0e901a92a709f5"} +{"dateoftransformation":"2018-09-13","originalId":["corda_______::999838074"],"collectedfrom":[{"value":"CORDA - COmmon Research DAta Warehouse","key":"10|openaire____::b30dac7baac631f3da7c2bb18dd9891f"}],"organization":{"metadata":{"eclegalbody":{"value":"true"},"eclegalperson":{"value":"true"},"ecinternationalorganization":{"value":"false"},"legalshortname":{"value":"UNIVR"},"ecresearchorganization":{"value":"true"},"ecnonprofit":{"value":"true"},"ecenterprise":{"value":"false"},"websiteurl":{"value":"http://www.univr.it"},"ecnutscode":{"value":"false"},"ecinternationalorganizationeurinterests":{"value":"false"},"legalname":{"value":"GENOVA"},"country":{"classid":"IT","classname":"IT","schemename":"dnet:countries","schemeid":"dnet:countries"},"echighereducation":{"value":"true"}}},"dateofcollection":"2018-03-12","type":20,"id":"20|corda_______::3898c35de19616484a0e901a92a709f5"} \ No newline at end of file diff --git a/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/result.simple.pace.conf b/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/result.simple.pace.conf index 0249fb8..3f13651 100644 --- a/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/result.simple.pace.conf +++ b/dnet-dedup-test/src/main/resources/eu/dnetlib/pace/result.simple.pace.conf @@ -1,7 +1,7 @@ { "wf" : { "threshold" : "0.99", - "run" : "001", + "dedupRun" : "001", "entityType" : "result", "orderField" : "title", "queueMaxSize" : "2000", diff --git a/dnet-dedup-test/src/test/java/eu/dnetlib/pace/AbstractProtoPaceTest.java b/dnet-dedup-test/src/test/java/eu/dnetlib/pace/AbstractProtoPaceTest.java index 30be387..7adfa5c 100644 --- a/dnet-dedup-test/src/test/java/eu/dnetlib/pace/AbstractProtoPaceTest.java +++ b/dnet-dedup-test/src/test/java/eu/dnetlib/pace/AbstractProtoPaceTest.java @@ -1,208 +1,212 @@ -//package eu.dnetlib.pace; -// -//import com.google.common.collect.Lists; -//import com.google.common.collect.Sets; -//import com.google.gson.Gson; -//import eu.dnetlib.data.proto.FieldTypeProtos.Author; -//import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier; -//import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty; -//import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty.Builder; -//import eu.dnetlib.data.proto.OafProtos.Oaf; -//import eu.dnetlib.data.proto.OafProtos.OafEntity; -//import eu.dnetlib.data.proto.OrganizationProtos.Organization; -//import eu.dnetlib.data.proto.ResultProtos.Result; -//import eu.dnetlib.pace.config.Config; -//import eu.dnetlib.pace.config.DedupConfig; -//import eu.dnetlib.pace.config.Type; -//import eu.dnetlib.pace.model.Field; -//import eu.dnetlib.pace.model.FieldValueImpl; -//import eu.dnetlib.pace.model.MapDocument; -//import eu.dnetlib.pace.model.ProtoDocumentBuilder; -//import eu.dnetlib.pace.model.gt.GTAuthor; -//import org.apache.commons.io.IOUtils; -//import org.apache.commons.lang.RandomStringUtils; -//import org.apache.commons.lang.StringUtils; -//import org.apache.commons.lang3.RandomUtils; -// -//import java.io.IOException; -//import java.io.StringWriter; -//import java.util.ArrayList; -//import java.util.LinkedList; -//import java.util.List; -//import java.util.Set; -//import java.util.stream.Collectors; -//import java.util.stream.IntStream; -// -//public abstract class AbstractProtoPaceTest extends OafTest { -// -// protected DedupConfig getResultFullConf() { -// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.full.pace.conf")); -// } -// -// protected DedupConfig getResultSimpleConf() { -// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.simple.pace.conf")); -// } -// -// protected DedupConfig getResultConf() { -// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.pace.conf")); -// } -// -// protected DedupConfig getOrganizationSimpleConf() { -// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf")); -// } -// -// protected DedupConfig getResultAuthorsConf() { -// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.authors.pace.conf")); -// } -// -// protected DedupConfig getResultProdConf() { -// return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.prod.pace.conf")); -// } -// -// protected MapDocument author(final Config conf, final String id, final Oaf oaf) { -// return ProtoDocumentBuilder.newInstance(id, oaf.getEntity(), conf.model()); -// } -// -// protected GTAuthor getGTAuthor(final String path) { -// -// final Gson gson = new Gson(); -// -// final String json = readFromClasspath(path); -// -// final GTAuthor gta = gson.fromJson(json, GTAuthor.class); -// -// return gta; -// } -// -// protected String readFromClasspath(final String filename) { -// final StringWriter sw = new StringWriter(); -// try { -// IOUtils.copy(getClass().getResourceAsStream(filename), sw); -// return sw.toString(); -// } catch (final IOException e) { -// throw new RuntimeException("cannot load resource from classpath: " + filename); -// } -// } -// -// protected MapDocument result(final Config config, final String id, final String title) { -// return result(config, id, title, null, new ArrayList<>(), null); -// } -// -// protected MapDocument result(final Config config, final String id, final String title, final String date) { -// return result(config, id, title, date, new ArrayList<>(), null); -// } -// -// protected MapDocument result(final Config config, final String id, final String title, final String date, final List pid) { -// return result(config, id, title, date, pid, null); -// } -// -// protected MapDocument result(final Config config, final String id, final String title, final String date, final String pid) { -// return result(config, id, title, date, pid, null); -// } -// -// protected MapDocument result(final Config config, final String id, final String title, final String date, final String pid, final List authors) { -// return result(config, id, title, date, Lists.newArrayList(pid), authors); -// } -// -// static List pidTypes = Lists.newArrayList(); -// static { -// pidTypes.add("doi"); -// //pidTypes.add("oai"); -// //pidTypes.add("pmid"); -// } -// -// protected MapDocument result(final Config config, final String id, final String title, final String date, final List pid, final List authors) { -// final Result.Metadata.Builder metadata = Result.Metadata.newBuilder(); -// if (!StringUtils.isBlank(title)) { -// metadata.addTitle(getStruct(title, getQualifier("main title", "dnet:titles"))); -// metadata.addTitle(getStruct(RandomStringUtils.randomAlphabetic(10), getQualifier("alternative title", "dnet:titles"))); -// } -// if (!StringUtils.isBlank(date)) { -// metadata.setDateofacceptance(sf(date)); -// } -// -// final OafEntity.Builder entity = oafEntity(id, eu.dnetlib.data.proto.TypeProtos.Type.result); -// final Result.Builder result = Result.newBuilder().setMetadata(metadata); -// -// if (authors != null) { -// result.getMetadataBuilder().addAllAuthor( -// IntStream.range(0, authors.size()) -// .mapToObj(i -> author(authors.get(i), i)) -// .collect(Collectors.toCollection(LinkedList::new))); -// } -// -// entity.setResult(result); -// -// if (pid != null) { -// for(String p : pid) { -// if (!StringUtils.isBlank(p)) { -// entity.addPid(sp(p, pidTypes.get(RandomUtils.nextInt(0, pidTypes.size() - 1)))); -// //entity.addPid(sp(RandomStringUtils.randomAlphabetic(10), "oai")); -// } -// } -// } -// -// final OafEntity build = entity.build(); -// return ProtoDocumentBuilder.newInstance(id, build, config.model()); -// } -// -// private Author author(final String s, int rank) { -// final eu.dnetlib.pace.model.Person p = new eu.dnetlib.pace.model.Person(s, false); -// final Author.Builder author = Author.newBuilder(); -// if (p.isAccurate()) { -// author.setName(p.getNormalisedFirstName()); -// author.setSurname(p.getNormalisedSurname()); -// } -// author.setFullname(p.getNormalisedFullname()); -// author.setRank(rank); -// -// return author.build(); -// } -// -// private OafEntity.Builder oafEntity(final String id, final eu.dnetlib.data.proto.TypeProtos.Type type) { -// final OafEntity.Builder entity = OafEntity.newBuilder().setId(id).setType(type); -// return entity; -// } -// -// protected MapDocument organization(final Config config, final String id, final String legalName) { -// return organization(config, id, legalName, null); -// } -// -// protected MapDocument organization(final Config config, final String id, final String legalName, final String legalShortName) { -// final Organization.Metadata.Builder metadata = Organization.Metadata.newBuilder(); -// if (legalName != null) { -// metadata.setLegalname(sf(legalName)); -// } -// if (legalShortName != null) { -// metadata.setLegalshortname(sf(legalShortName)); -// } -// -// final OafEntity.Builder entity = oafEntity(id, eu.dnetlib.data.proto.TypeProtos.Type.result); -// entity.setOrganization(Organization.newBuilder().setMetadata(metadata)); -// -// return ProtoDocumentBuilder.newInstance(id, entity.build(), config.model()); -// } -// -// private StructuredProperty sp(final String pid, final String type) { -// final Builder pidSp = -// StructuredProperty.newBuilder().setValue(pid) -// .setQualifier(Qualifier.newBuilder().setClassid(type).setClassname(type).setSchemeid("dnet:pid_types").setSchemename("dnet:pid_types")); -// return pidSp.build(); -// } -// -// protected Field title(final String s) { -// return new FieldValueImpl(Type.String, "title", s); -// } -// -// protected static Builder getStruct(final String value, final Qualifier.Builder qualifier) { -// return StructuredProperty.newBuilder().setValue(value).setQualifier(qualifier); -// } -// -// /* -// * protected static StringField.Builder sf(final String s) { return StringField.newBuilder().setValue(s); } -// * -// * protected static Qualifier.Builder getQualifier(final String classname, final String schemename) { return -// * Qualifier.newBuilder().setClassid(classname).setClassname(classname).setSchemeid(schemename).setSchemename(schemename); } -// */ -// -//} +package eu.dnetlib.pace; + +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import eu.dnetlib.data.proto.FieldTypeProtos.Author; +import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier; +import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty; +import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty.Builder; +import eu.dnetlib.data.proto.OafProtos.Oaf; +import eu.dnetlib.data.proto.OafProtos.OafEntity; +import eu.dnetlib.data.proto.OrganizationProtos.Organization; +import eu.dnetlib.data.proto.ResultProtos.Result; +import eu.dnetlib.pace.config.Config; +import eu.dnetlib.pace.config.DedupConfig; +import eu.dnetlib.pace.config.Type; +import eu.dnetlib.pace.model.Field; +import eu.dnetlib.pace.model.FieldValueImpl; +import eu.dnetlib.pace.model.MapDocument; +import eu.dnetlib.pace.model.ProtoDocumentBuilder; +import eu.dnetlib.pace.model.gt.GTAuthor; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.RandomUtils; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public abstract class AbstractProtoPaceTest extends OafTest { + + protected DedupConfig getResultFullConf() { + return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.full.pace.conf")); + } + + protected DedupConfig getResultSimpleConf() { + return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.simple.pace.conf")); + } + + protected DedupConfig getResultConf() { + return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.pace.conf")); + } + + protected DedupConfig getOrganizationSimpleConf() { + return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf")); + } + + + protected DedupConfig getOrganizationTestConf() { + return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.test.conf")); + } + + + protected DedupConfig getResultAuthorsConf() { + return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.authors.pace.conf")); + } + + protected DedupConfig getResultProdConf() { + return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.prod.pace.conf")); + } + + protected MapDocument author(final Config conf, final String id, final Oaf oaf) { + return ProtoDocumentBuilder.newInstance(id, oaf.getEntity(), conf.model()); + } + + protected GTAuthor getGTAuthor(final String path) { + + final Gson gson = new Gson(); + + final String json = readFromClasspath(path); + + final GTAuthor gta = gson.fromJson(json, GTAuthor.class); + + return gta; + } + + protected String readFromClasspath(final String filename) { + final StringWriter sw = new StringWriter(); + try { + IOUtils.copy(getClass().getResourceAsStream(filename), sw); + return sw.toString(); + } catch (final IOException e) { + throw new RuntimeException("cannot load resource from classpath: " + filename); + } + } + + protected MapDocument result(final Config config, final String id, final String title) { + return result(config, id, title, null, new ArrayList<>(), null); + } + + protected MapDocument result(final Config config, final String id, final String title, final String date) { + return result(config, id, title, date, new ArrayList<>(), null); + } + + protected MapDocument result(final Config config, final String id, final String title, final String date, final List pid) { + return result(config, id, title, date, pid, null); + } + + protected MapDocument result(final Config config, final String id, final String title, final String date, final String pid) { + return result(config, id, title, date, pid, null); + } + + protected MapDocument result(final Config config, final String id, final String title, final String date, final String pid, final List authors) { + return result(config, id, title, date, Lists.newArrayList(pid), authors); + } + + static List pidTypes = Lists.newArrayList(); + static { + pidTypes.add("doi"); + //pidTypes.add("oai"); + //pidTypes.add("pmid"); + } + + protected MapDocument result(final Config config, final String id, final String title, final String date, final List pid, final List authors) { + final Result.Metadata.Builder metadata = Result.Metadata.newBuilder(); + if (!StringUtils.isBlank(title)) { + metadata.addTitle(getStruct(title, getQualifier("main title", "dnet:titles"))); + metadata.addTitle(getStruct(RandomStringUtils.randomAlphabetic(10), getQualifier("alternative title", "dnet:titles"))); + } + if (!StringUtils.isBlank(date)) { + metadata.setDateofacceptance(sf(date)); + } + + final OafEntity.Builder entity = oafEntity(id, eu.dnetlib.data.proto.TypeProtos.Type.result); + final Result.Builder result = Result.newBuilder().setMetadata(metadata); + + if (authors != null) { + result.getMetadataBuilder().addAllAuthor( + IntStream.range(0, authors.size()) + .mapToObj(i -> author(authors.get(i), i)) + .collect(Collectors.toCollection(LinkedList::new))); + } + + entity.setResult(result); + + if (pid != null) { + for(String p : pid) { + if (!StringUtils.isBlank(p)) { + entity.addPid(sp(p, pidTypes.get(RandomUtils.nextInt(0, pidTypes.size() - 1)))); + //entity.addPid(sp(RandomStringUtils.randomAlphabetic(10), "oai")); + } + } + } + + final OafEntity build = entity.build(); + return ProtoDocumentBuilder.newInstance(id, build, config.model()); + } + + private Author author(final String s, int rank) { + final eu.dnetlib.pace.model.Person p = new eu.dnetlib.pace.model.Person(s, false); + final Author.Builder author = Author.newBuilder(); + if (p.isAccurate()) { + author.setName(p.getNormalisedFirstName()); + author.setSurname(p.getNormalisedSurname()); + } + author.setFullname(p.getNormalisedFullname()); + author.setRank(rank); + + return author.build(); + } + + private OafEntity.Builder oafEntity(final String id, final eu.dnetlib.data.proto.TypeProtos.Type type) { + final OafEntity.Builder entity = OafEntity.newBuilder().setId(id).setType(type); + return entity; + } + + protected MapDocument organization(final Config config, final String id, final String legalName) { + return organization(config, id, legalName, null); + } + + protected MapDocument organization(final Config config, final String id, final String legalName, final String legalShortName) { + final Organization.Metadata.Builder metadata = Organization.Metadata.newBuilder(); + if (legalName != null) { + metadata.setLegalname(sf(legalName)); + } + if (legalShortName != null) { + metadata.setLegalshortname(sf(legalShortName)); + } + + final OafEntity.Builder entity = oafEntity(id, eu.dnetlib.data.proto.TypeProtos.Type.result); + entity.setOrganization(Organization.newBuilder().setMetadata(metadata)); + + return ProtoDocumentBuilder.newInstance(id, entity.build(), config.model()); + } + + private StructuredProperty sp(final String pid, final String type) { + final Builder pidSp = + StructuredProperty.newBuilder().setValue(pid) + .setQualifier(Qualifier.newBuilder().setClassid(type).setClassname(type).setSchemeid("dnet:pid_types").setSchemename("dnet:pid_types")); + return pidSp.build(); + } + + protected Field title(final String s) { + return new FieldValueImpl(Type.String, "title", s); + } + + protected static Builder getStruct(final String value, final Qualifier.Builder qualifier) { + return StructuredProperty.newBuilder().setValue(value).setQualifier(qualifier); + } + + /* + * protected static StringField.Builder sf(final String s) { return StringField.newBuilder().setValue(s); } + * + * protected static Qualifier.Builder getQualifier(final String classname, final String schemename) { return + * Qualifier.newBuilder().setClassid(classname).setClassname(classname).setSchemeid(schemename).setSchemename(schemename); } + */ + +} diff --git a/dnet-dedup-test/src/test/java/eu/dnetlib/pace/OafTest.java b/dnet-dedup-test/src/test/java/eu/dnetlib/pace/OafTest.java index 3ec495b..590c416 100644 --- a/dnet-dedup-test/src/test/java/eu/dnetlib/pace/OafTest.java +++ b/dnet-dedup-test/src/test/java/eu/dnetlib/pace/OafTest.java @@ -1,446 +1,446 @@ -//package eu.dnetlib.pace; -// -//import com.google.protobuf.GeneratedMessage; -//import com.google.protobuf.InvalidProtocolBufferException; -//import eu.dnetlib.data.mapreduce.util.OafDecoder; -//import eu.dnetlib.data.proto.DatasourceOrganizationProtos.DatasourceOrganization; -//import eu.dnetlib.data.proto.DatasourceOrganizationProtos.DatasourceOrganization.Provision; -//import eu.dnetlib.data.proto.DatasourceProtos.Datasource; -//import eu.dnetlib.data.proto.DedupProtos.Dedup; -//import eu.dnetlib.data.proto.FieldTypeProtos.*; -//import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty.Builder; -//import eu.dnetlib.data.proto.KindProtos.Kind; -//import eu.dnetlib.data.proto.OafProtos.Oaf; -//import eu.dnetlib.data.proto.OafProtos.OafEntity; -//import eu.dnetlib.data.proto.OafProtos.OafRel; -//import eu.dnetlib.data.proto.OrganizationOrganizationProtos.OrganizationOrganization; -//import eu.dnetlib.data.proto.OrganizationProtos.Organization; -//import eu.dnetlib.data.proto.ProjectOrganizationProtos.ProjectOrganization; -//import eu.dnetlib.data.proto.ProjectOrganizationProtos.ProjectOrganization.Participation; -//import eu.dnetlib.data.proto.ProjectProtos.Project; -//import eu.dnetlib.data.proto.RelMetadataProtos.RelMetadata; -//import eu.dnetlib.data.proto.RelTypeProtos.RelType; -//import eu.dnetlib.data.proto.RelTypeProtos.SubRelType; -//import eu.dnetlib.data.proto.ResultProjectProtos.ResultProject; -//import eu.dnetlib.data.proto.ResultProjectProtos.ResultProject.Outcome; -//import eu.dnetlib.data.proto.ResultProtos.Result; -//import eu.dnetlib.data.proto.ResultProtos.Result.Context; -//import eu.dnetlib.data.proto.ResultProtos.Result.Instance; -//import eu.dnetlib.data.proto.ResultResultProtos.ResultResult; -//import eu.dnetlib.data.proto.ResultResultProtos.ResultResult.Similarity; -//import eu.dnetlib.data.proto.TypeProtos.Type; -// -//public class OafTest { -// -// public static final String CITATION_JSON = -// "\n \n [10] M. Foret et al., Phys. Rev. B 66, 024204 (2002).\n \n \n [11] B. Ru\175404\264e et al., Phys. Rev. Lett. 90, 095502 (2003).\n \n \n [12] U. Buchenau et al., Phys. Rev. B 34, 5665 (1986).\n \n \n [13] S.N. Taraskin and S.R. Elliott, J. Phys.: Condens. Mat- ter 11, A219 (1999).\n \n \n [14] B. Hehlen et al., Phys. Rev. Lett. 84, 5355 (2000).\n \n \n [15] N.V. Surotsev et al., J. Phys.: Condens. Matter 10, L113 (1998).\n \n \n [16] D.A. Parshin and C. Laermans, Phys. Rev. B 63, 132203 (2001).\n \n \n [17] V.L. Gurevich et al., Phys. Rev. B 67, 094203 (2003).\n \n \n [18] A. Matic et al., Phys. Rev. Lett. 86, 3803 (2001).\n \n \n [19] E. Rat et al., arXiv:cond-mat/0505558, 23 May 2005.\n \n \n [1] R.C. Zeller and R.O. Pohl, Phys. Rev. B 4, 2029 (1971).\n \n \n [20] C.A. Angell, J. Non-Cryst. Solids 131\20023133, 13 (1991).\n \n \n [21] A.P. Sokolov et al., Phys. Rev. Lett. 71, 2062 (1993).\n \n \n [22] T. Matsuo et al., Solid State Ionics 154-155, 759 (2002).\n \n \n [23] V.K. Malinovsky et al., Europhys. Lett. 11, 43 (1990).\n \n \n [24] J. Lor\250osch et al., J. Non-Cryst. Solids 69, 1 (1984).\n \n \n [25] U. Buchenau, Z. Phys. B 58, 181 (1985).\n \n \n [26] A.F. Io\175400e and A.R. Regel, Prog. Semicond. 4, 237 (1960).\n \n \n [27] R. Dell\20031Anna et al., Phys. Rev. Lett. 80, 1236 (1998).\n \n \n [28] D. Fioretto et al., Phys. Rev. E 59, 4470 (1999).\n \n \n [29] U. Buchenau et al., Phys. Rev. Lett. 77, 4035 (1996).\n \n \n [2] M. Rothenfusser et al., Phys. Rev. B 27, 5196 (1983).\n \n \n [30] J. Mattsson et al., J. Phys.: Condens. Matter 15, S1259 (2003).\n \n \n [31] T. Scopigno et al., Phys. Rev. Lett. 92, 025503 (2004).\n \n \n [32] M. Foret et al., Phys. Rev. Lett. 81, 2100 (1998).\n \n \n [33] F. Sette et al., Science 280, 1550 (1998).\n \n \n [34] J. Wuttke et al., Phys. Rev. E 52, 4026 (1995).\n \n \n [35] M.A. Ramos et al., Phys. Rev. Lett. 78, 82 (1997).\n \n \n [36] G. Monaco et al., Phys. Rev. Lett. 80, 2161 (1998).\n \n \n [37] A. T\250olle, Rep. Prog. Phys. 64, 1473 (2001).\n \n \n [38] As the straight lines do not cross the origin, this does not 2 imply \1623 \21035 \1651 .\n \n \n [39] A. Matic et al., Europhys. Lett. 54, 77 (2001).\n \n \n [3] S. Hunklinger and W. Arnold, in Physical Acoustics, Vol. XII, W.P. Mason and R.N. Thurston Eds. (Academic Press, N.Y. 1976), p. 155.\n \n \n [40] IXS data are usually not available below \1651co, mostly for experimental reasons. E.g., that the rapid onset was not evidenced in vitreous silica [27], is not indicative of its absence but rather of a low qco \21074 1 nm\210221.\n \n \n [41] G. Ruocco et al., Phys. Rev. Lett. 83, 5583 (1999).\n \n \n [42] D. C\1307 iplys et al., J. Physique (Paris) 42, C6-184 (1981).\n \n \n [43] R. Vacher et al., Rev. Sci. Instrum. 51, 288 (1980).\n \n \n [44] R. Vacher et al., arXiv:cond-mat/0505560, 23 May 2005.\n \n \n [45] T.N. Claytor et al., Phys. Rev. B 18, 5842 (1978).\n \n \n [46] M. Arai et al., Physica B 263-264, 268 (1999).\n \n \n [4] R. Vacher et al., J. Non-Cryst. Solids 45, 397 (1981); T.C. Zhu et al., Phys. Rev. B 44, 4281 (1991).\n \n \n [5] J.E. Graebner et al., Phys. Rev. B 34, 5696 (1986).\n \n \n [6] E. Duval and A. Mermet, Phys. Rev. B 58, 8159 (1998).\n \n \n [7] A. Matic et al., Phys. Rev. Lett. 93, 145502 (2004).\n \n \n [8] Often alluded to, e.g. in the Encyclopedia of Materials: Science and Technology, K.H.J. Buschow et al., Eds., Vol. 1 (Elsevier, Oxford, 2001), articles by S.R. Elliott on pp. 171-174 and U. Buchenau on pp. 212-215.\n \n \n [9] E. Rat et al., Phys. Rev. Lett. 83, 1355 (1999).\n \n"; -// -// public static final String STATISTICS_JSON = -// "[{ \"citationsPerYear\": \"many\", \"anotherCoolStatistic\": \"WoW\", \"nestedStat\": { \"firstNestedStat\" : \"value 1\", \"secondNestedStat\" : \"value 2\"}, \"listingStat\" : [ \"one\", \"two\" ] }]"; -// -// public static Builder getStructuredproperty(final String value, final String classname, final String schemename) { -// return getStructuredproperty(value, classname, schemename, null); -// } -// -// public static Builder getStructuredproperty(final String value, final String classname, final String schemename, final DataInfo dataInfo) { -// final Builder sp = StructuredProperty.newBuilder().setValue(value).setQualifier(getQualifier(classname, schemename)); -// if (dataInfo != null) { -// sp.setDataInfo(dataInfo); -// } -// return sp; -// } -// -// public static Qualifier.Builder getQualifier(final String classname, final String schemename) { -// return Qualifier.newBuilder().setClassid(classname).setClassname(classname).setSchemeid(schemename).setSchemename(schemename); -// } -// -// public static KeyValue getKV(final String id, final String name) { -// return KeyValue.newBuilder().setKey(id).setValue(name).build(); -// } -// -// public static OafEntity getDatasource(final String datasourceId) { -// return OafEntity -// .newBuilder() -// .setType(Type.datasource) -// .setId(datasourceId) -// .setDatasource( -// Datasource.newBuilder().setMetadata( -// Datasource.Metadata.newBuilder().setOfficialname(sf("officialname")).setEnglishname(sf("englishname")) -// .setWebsiteurl(sf("websiteurl")).setContactemail(sf("contactemail")).addAccessinfopackage(sf("accessinforpackage")) -// .setNamespaceprefix(sf("namespaceprofix")).setDescription(sf("description")).setOdnumberofitems(sf("numberofitems")) -// .setOdnumberofitemsdate(sf("numberofitems date")) -// // .addOdsubjects("subjects") -// .setOdpolicies(sf("policies")).addOdlanguages(sf("languages")).addOdcontenttypes(sf("contenttypes")) -// .setDatasourcetype(getQualifier("type class", "type scheme")))).build(); -// } -// -// public static OafEntity getResult(final String id) { -// return getResultBuilder(id).build(); -// } -// -// public static OafEntity.Builder getResultBuilder(final String id) { -// return OafEntity -// .newBuilder() -// .setType(Type.result) -// .setId(id) -// .setResult( -// Result.newBuilder() -// .setMetadata( -// Result.Metadata -// .newBuilder() -// .addTitle( -// getStructuredproperty( -// "Analysis of cell viability in intervertebral disc: Effect of endplate permeability on cell population", -// "main title", "dnet:result_titles", getDataInfo())) -// .addTitle(getStructuredproperty("Another title", "alternative title", "dnet:result_titles", getDataInfo())) -// .addSubject(getStructuredproperty("Biophysics", "subject", "dnet:result_sujects")) -// .setDateofacceptance(sf("2010-01-01")).addSource(sf("sourceA")).addSource(sf("sourceB")) -// .addContext(Context.newBuilder().setId("egi::virtual::970")) -// .addContext(Context.newBuilder().setId("egi::classification::natsc::math::applied")) -// .addContext(Context.newBuilder().setId("egi::classification::natsc::math")) -// .addContext(Context.newBuilder().setId("egi::classification::natsc")) -// .addContext(Context.newBuilder().setId("egi::classification")).addContext(Context.newBuilder().setId("egi")) -// .addDescription(sf("Responsible for making and maintaining the extracellular matrix ...")) -// .addDescription(sf("Another description ...")).setPublisher(sf("ELSEVIER SCI LTD")) -// .setResulttype(getQualifier("publication", "dnet:result_types")) -// .setLanguage(getQualifier("eng", "dnet:languages"))).addInstance(getInstance("10|od__10", "Uk pubmed")) -// .addInstance(getInstance("10|od__10", "arxiv"))) -// .addCollectedfrom(getKV("opendoar____::1064", "Oxford University Research Archive")) -// .addPid(getStructuredproperty("doi:74293", "doi", "dnet:pids")).addPid(getStructuredproperty("oai:74295", "oai", "dnet:pids")) -// .setDateofcollection(""); -// } -// -// public static DataInfo getDataInfo() { -// return getDataInfo("0.4"); -// } -// -// public static DataInfo getDataInfo(final String trust) { -// return DataInfo.newBuilder().setDeletedbyinference(false).setTrust("0.4").setInferenceprovenance("algo").setProvenanceaction(getQualifier("xx", "yy")) -// .build(); -// } -// -// public static Instance.Builder getInstance(final String hostedbyId, final String hostedbyName) { -// return Instance.newBuilder().setHostedby(getKV(hostedbyId, hostedbyName)).setAccessright(getQualifier("OpenAccess", "dnet:access_modes")) -// .setInstancetype(getQualifier("publication", "dnet:result_typologies")).addUrl("webresource url"); -// -// } -// -// public static OafRel getDedupRel(final String source, final String target, final RelType relType, final String relClass) { -// return OafRel.newBuilder().setSource(source).setTarget(target).setRelType(relType).setSubRelType(SubRelType.dedup).setRelClass(relClass) -// .setChild(false).setCachedTarget(getResult(target)) -// .setResultResult(ResultResult.newBuilder().setDedup(Dedup.newBuilder().setRelMetadata(RelMetadata.getDefaultInstance()))) -// .build(); -// } -// -// public static OafRel getProjectOrganization(final String source, final String target, final String relClass) throws InvalidProtocolBufferException { -// final OafRel.Builder oafRel = OafRel -// .newBuilder() -// .setSource(source) -// .setTarget(target) -// .setRelType(RelType.projectOrganization) -// .setSubRelType(SubRelType.participation) -// .setRelClass(relClass) -// .setChild(false) -// .setProjectOrganization( -// ProjectOrganization.newBuilder().setParticipation( -// Participation.newBuilder().setParticipantnumber("" + 1) -// .setRelMetadata(relMetadata(relClass, "dnet:project_organization_relations")))); -// switch (Participation.RelName.valueOf(relClass)) { -// case hasParticipant: -// oafRel.setCachedTarget(getProjectFP7(target, "SP3")); -// break; -// case isParticipant: -// oafRel.setCachedTarget(getOrganization(target)); -// break; -// default: -// break; -// } -// return oafRel.build(); -// } -// -// public static GeneratedMessage getOrganizationOrganization(final String source, final String target, final String relClass) { -// final OafRel.Builder oafRel = OafRel -// .newBuilder() -// .setSource(source) -// .setTarget(target) -// .setRelType(RelType.organizationOrganization) -// .setSubRelType(SubRelType.dedup) -// .setRelClass(relClass) -// .setChild(true) -// .setOrganizationOrganization( -// OrganizationOrganization.newBuilder().setDedup( -// Dedup.newBuilder().setRelMetadata(relMetadata(relClass, "dnet:organization_organization_relations")))); -// -// switch (Dedup.RelName.valueOf(relClass)) { -// case isMergedIn: -// oafRel.setCachedTarget(getOrganization(source)); -// break; -// case merges: -// oafRel.setCachedTarget(getOrganization(target)); -// break; -// default: -// break; -// } -// return oafRel.build(); -// } -// -// public static OafRel getDatasourceOrganization(final String source, final String target, final String relClass) throws InvalidProtocolBufferException { -// final OafRel.Builder oafRel = OafRel -// .newBuilder() -// .setSource(source) -// .setTarget(target) -// .setRelType(RelType.datasourceOrganization) -// .setSubRelType(SubRelType.provision) -// .setRelClass(relClass) -// .setChild(false) -// .setDatasourceOrganization( -// DatasourceOrganization.newBuilder().setProvision( -// Provision.newBuilder().setRelMetadata(relMetadata(relClass, "dnet:datasource_organization_relations")))); -// switch (Provision.RelName.valueOf(relClass)) { -// case isProvidedBy: -// oafRel.setCachedTarget(getOrganization(target)); -// break; -// case provides: -// oafRel.setCachedTarget(getDatasource(target)); -// break; -// default: -// break; -// } -// return oafRel.build(); -// } -// -// public static OafRel getSimilarityRel(final String sourceId, final String targetId, final OafEntity result, final String relClass) { -// return OafRel -// .newBuilder() -// .setSource(sourceId) -// .setTarget(targetId) -// .setRelType(RelType.resultResult) -// .setSubRelType(SubRelType.similarity) -// .setRelClass(relClass) -// .setChild(false) -// .setCachedTarget(result) -// .setResultResult( -// ResultResult.newBuilder().setSimilarity( -// Similarity.newBuilder().setRelMetadata(relMetadata(relClass, "dnet:resultResult_relations")).setSimilarity(.4f) -// .setType(Similarity.Type.STANDARD))).build(); -// } -// -// public static RelMetadata.Builder relMetadata(final String classname, final String schemename) { -// return RelMetadata.newBuilder().setSemantics(getQualifier(classname, schemename)); -// } -// -// public static OafEntity getOrganization(final String orgId) { -// return OafEntity -// .newBuilder() -// .setType(Type.organization) -// .setId(orgId) -// .addCollectedfrom(getKV("opendoar_1234", "UK pubmed")) -// .setOrganization( -// Organization.newBuilder().setMetadata( -// Organization.Metadata.newBuilder().setLegalname(sf("CENTRE D'APPUI A LA RECHERCHE ET A LA FORMATION GIE")) -// .setLegalshortname(sf("CAREF")).setWebsiteurl(sf("www.caref-mali.org")) -// .setCountry(getQualifier("ML", "dnet:countries")))).build(); -// } -// -// public static OafRel getResultProject(final String from, final String to, final OafEntity project, final String relClass) -// throws InvalidProtocolBufferException { -// return OafRel -// .newBuilder() -// .setSource(from) -// .setTarget(to) -// .setRelType(RelType.resultProject) -// .setSubRelType(SubRelType.outcome) -// .setRelClass(relClass) -// .setChild(false) -// .setResultProject( -// ResultProject.newBuilder().setOutcome(Outcome.newBuilder().setRelMetadata(relMetadata(relClass, "dnet:result_project_relations")))) -// .setCachedTarget(project).build(); -// } -// -// public static OafEntity getProjectFP7(final String projectId, final String fundingProgram) throws InvalidProtocolBufferException { -// return OafEntity -// .newBuilder() -// .setType(Type.project) -// .setId(projectId) -// .addCollectedfrom(getKV("opendoar_1234", "UK pubmed")) -// .setProject( -// Project.newBuilder() -// .setMetadata( -// Project.Metadata -// .newBuilder() -// .setAcronym(sf("5CYRQOL")) -// .setTitle(sf("Cypriot Researchers Contribute to our Quality of Life")) -// .setStartdate(sf("2007-05-01")) -// .setEnddate(sf("2007-10-31")) -// .setEcsc39(sf("false")) -// .setContracttype(getQualifier("CSA", "ec:FP7contractTypes")) -// .addFundingtree( -// sf("ec__________::ECECEuropean Commissionec__________::EC::FP7::" -// + fundingProgram -// + "::PEOPLEMarie-Curie ActionsPEOPLEec:programec__________::EC::FP7::" -// + fundingProgram -// + "" -// + fundingProgram -// + "-People" -// + fundingProgram -// + "ec:specificprogramec__________::EC::FP7SEVENTH FRAMEWORK PROGRAMMEFP7ec:frameworkprogram")))) -// .build(); -// } -// -// public static OafEntity getProjectWT() throws InvalidProtocolBufferException { -// return OafEntity -// .newBuilder() -// .setType(Type.project) -// .setId("project|wt::087536") -// .addCollectedfrom(getKV("wellcomeTrust", "wellcome trust")) -// .setProject( -// Project.newBuilder() -// .setMetadata( -// Project.Metadata -// .newBuilder() -// .setAcronym(sf("UNKNOWN")) -// .setTitle(sf("Research Institute for Infectious Diseases of Poverty (IIDP).")) -// .setStartdate(sf("2007-05-01")) -// .setEnddate(sf("2007-10-31")) -// .setEcsc39(sf("false")) -// .setContracttype(getQualifier("UNKNOWN", "wt:contractTypes")) -// .addFundingtree( -// sf("wt__________::WTWTWellcome Trustwt__________::WT::UNKNOWNUNKNOWNUNKNOWNwt:fundingStream")) -// .addFundingtree( -// sf("wt__________::WTWTWellcome Trustwt__________::WT::Technology TransferTechnology TransferTechnology Transferwt:fundingStream")))) -// .build(); -// } -// -// public static ExtraInfo extraInfo(final String name, final String provenance, final String trust, final String typology, final String value) { -// final ExtraInfo.Builder e = ExtraInfo.newBuilder().setName(name).setProvenance(provenance).setTrust(trust).setTypology(typology).setValue(value); -// return e.build(); -// } -// -// // public static DocumentClasses documentClasses() { -// // DocumentClasses.Builder builder = DocumentClasses.newBuilder(); -// // for (int i = 0; i < RandomUtils.nextInt(N_DOCUMENT_CLASSES) + 1; i++) { -// // builder.addArXivClasses(getDocumentClass()).addDdcClasses(getDocumentClass()).addWosClasses(getDocumentClass()) -// // .addMeshEuroPMCClasses(getDocumentClass()); -// // } -// // return builder.build(); -// // } -// // -// // private static DocumentClass getDocumentClass() { -// // DocumentClass.Builder builder = DocumentClass.newBuilder(); -// // for (int i = 0; i < RandomUtils.nextInt(N_DOCUMENT_CLASS_LABELS) + 1; i++) { -// // builder.addClassLabels("test_class_" + i); -// // } -// // return builder.setConfidenceLevel(0.5F).build(); -// // } -// // -// // public static DocumentStatistics documentStatistics() { -// // return -// // DocumentStatistics.newBuilder().setCitationsFromAllPapers(basicCitationStatistics()).setCitationsFromPublishedPapers(basicCitationStatistics()) -// // .build(); -// // } -// // -// // private static BasicCitationStatistics basicCitationStatistics() { -// // BasicCitationStatistics.Builder builder = BasicCitationStatistics.newBuilder(); -// // for (int i = 0; i < N_CITATION_STATS; i++) { -// // builder.addNumberOfCitationsPerYear(statisticsKeyValue()); -// // builder.setNumberOfCitations(RandomUtils.nextInt(5) + 1); -// // } -// // return builder.build(); -// // } -// // -// // private static StatisticsKeyValue statisticsKeyValue() { -// // return StatisticsKeyValue.newBuilder().setKey((RandomUtils.nextInt(30) + 1980) + "").setValue(RandomUtils.nextInt(5) + 1).build(); -// // } -// // -// // public static AuthorStatistics authorStatistics() { -// // AuthorStatistics.Builder builder = AuthorStatistics.newBuilder(); -// // builder.setCore(commonCoreStatistics()); -// // for (int i = 0; i < N_COAUTHORS; i++) { -// // builder.addCoAuthors(coAuthor()); -// // } -// // return builder.build(); -// // } -// // -// // private static CoAuthor coAuthor() { -// // CoAuthor.Builder builder = CoAuthor.newBuilder(); -// // builder.setId("30|od______2345::" + Hashing.md5(RandomStringUtils.random(10))); -// // builder.setCoauthoredPapersCount(RandomUtils.nextInt(5) + 1); -// // return builder.build(); -// // } -// // -// // public static CommonCoreStatistics commonCoreStatistics() { -// // CommonCoreStatistics.Builder builder = CommonCoreStatistics.newBuilder(); -// // -// // builder.setAllPapers(coreStatistics()); -// // builder.setPublishedPapers(coreStatistics()); -// // -// // return builder.build(); -// // } -// // -// // private static CoreStatistics coreStatistics() { -// // CoreStatistics.Builder builder = CoreStatistics.newBuilder(); -// // -// // builder.setNumberOfPapers(RandomUtils.nextInt(10)); -// // builder.setCitationsFromAllPapers(extendedStatistics()); -// // builder.setCitationsFromPublishedPapers(extendedStatistics()); -// // -// // return builder.build(); -// // } -// // -// // private static ExtendedStatistics extendedStatistics() { -// // ExtendedStatistics.Builder builder = ExtendedStatistics.newBuilder(); -// // -// // builder.setBasic(basicCitationStatistics()); -// // builder.setAverageNumberOfCitationsPerPaper(RandomUtils.nextFloat()); -// // for (int i = 0; i < N_CITATION_STATS; i++) { -// // builder.addNumberOfPapersCitedAtLeastXTimes(statisticsKeyValue()); -// // } -// // -// // return builder.build(); -// // } -// -// public static StringField sf(final String s) { -// return sf(s, null); -// } -// -// public static StringField sf(final String s, final DataInfo dataInfo) { -// final StringField.Builder sf = StringField.newBuilder().setValue(s); -// if (dataInfo != null) { -// sf.setDataInfo(dataInfo); -// } -// return sf.build(); -// } -// -// public static OafDecoder embed(final GeneratedMessage msg, -// final Kind kind, -// final boolean deletedByInference, -// final boolean inferred, -// final String provenance, -// final String action) { -// -// final Oaf.Builder oaf = Oaf -// .newBuilder() -// .setKind(kind) -// .setLastupdatetimestamp(System.currentTimeMillis()) -// .setDataInfo( -// DataInfo.newBuilder().setDeletedbyinference(deletedByInference).setInferred(inferred).setTrust("0.5") -// .setInferenceprovenance(provenance).setProvenanceaction(getQualifier(action, action))); -// switch (kind) { -// case entity: -// oaf.setEntity((OafEntity) msg); -// break; -// case relation: -// oaf.setRel((OafRel) msg); -// break; -// default: -// break; -// } -// -// return OafDecoder.decode(oaf.build()); -// } -// -// public static OafDecoder embed(final GeneratedMessage msg, final Kind kind) { -// return embed(msg, kind, false, false, "inference_provenance", "provenance_action"); -// } -// -//} +package eu.dnetlib.pace; + +import com.google.protobuf.GeneratedMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import eu.dnetlib.data.mapreduce.util.OafDecoder; +import eu.dnetlib.data.proto.DatasourceOrganizationProtos.DatasourceOrganization; +import eu.dnetlib.data.proto.DatasourceOrganizationProtos.DatasourceOrganization.Provision; +import eu.dnetlib.data.proto.DatasourceProtos.Datasource; +import eu.dnetlib.data.proto.DedupProtos.Dedup; +import eu.dnetlib.data.proto.FieldTypeProtos.*; +import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty.Builder; +import eu.dnetlib.data.proto.KindProtos.Kind; +import eu.dnetlib.data.proto.OafProtos.Oaf; +import eu.dnetlib.data.proto.OafProtos.OafEntity; +import eu.dnetlib.data.proto.OafProtos.OafRel; +import eu.dnetlib.data.proto.OrganizationOrganizationProtos.OrganizationOrganization; +import eu.dnetlib.data.proto.OrganizationProtos.Organization; +import eu.dnetlib.data.proto.ProjectOrganizationProtos.ProjectOrganization; +import eu.dnetlib.data.proto.ProjectOrganizationProtos.ProjectOrganization.Participation; +import eu.dnetlib.data.proto.ProjectProtos.Project; +import eu.dnetlib.data.proto.RelMetadataProtos.RelMetadata; +import eu.dnetlib.data.proto.RelTypeProtos.RelType; +import eu.dnetlib.data.proto.RelTypeProtos.SubRelType; +import eu.dnetlib.data.proto.ResultProjectProtos.ResultProject; +import eu.dnetlib.data.proto.ResultProjectProtos.ResultProject.Outcome; +import eu.dnetlib.data.proto.ResultProtos.Result; +import eu.dnetlib.data.proto.ResultProtos.Result.Context; +import eu.dnetlib.data.proto.ResultProtos.Result.Instance; +import eu.dnetlib.data.proto.ResultResultProtos.ResultResult; +import eu.dnetlib.data.proto.ResultResultProtos.ResultResult.Similarity; +import eu.dnetlib.data.proto.TypeProtos.Type; + +public class OafTest { + + public static final String CITATION_JSON = + "\n \n [10] M. Foret et al., Phys. Rev. B 66, 024204 (2002).\n \n \n [11] B. Ru\175404\264e et al., Phys. Rev. Lett. 90, 095502 (2003).\n \n \n [12] U. Buchenau et al., Phys. Rev. B 34, 5665 (1986).\n \n \n [13] S.N. Taraskin and S.R. Elliott, J. Phys.: Condens. Mat- ter 11, A219 (1999).\n \n \n [14] B. Hehlen et al., Phys. Rev. Lett. 84, 5355 (2000).\n \n \n [15] N.V. Surotsev et al., J. Phys.: Condens. Matter 10, L113 (1998).\n \n \n [16] D.A. Parshin and C. Laermans, Phys. Rev. B 63, 132203 (2001).\n \n \n [17] V.L. Gurevich et al., Phys. Rev. B 67, 094203 (2003).\n \n \n [18] A. Matic et al., Phys. Rev. Lett. 86, 3803 (2001).\n \n \n [19] E. Rat et al., arXiv:cond-mat/0505558, 23 May 2005.\n \n \n [1] R.C. Zeller and R.O. Pohl, Phys. Rev. B 4, 2029 (1971).\n \n \n [20] C.A. Angell, J. Non-Cryst. Solids 131\20023133, 13 (1991).\n \n \n [21] A.P. Sokolov et al., Phys. Rev. Lett. 71, 2062 (1993).\n \n \n [22] T. Matsuo et al., Solid State Ionics 154-155, 759 (2002).\n \n \n [23] V.K. Malinovsky et al., Europhys. Lett. 11, 43 (1990).\n \n \n [24] J. Lor\250osch et al., J. Non-Cryst. Solids 69, 1 (1984).\n \n \n [25] U. Buchenau, Z. Phys. B 58, 181 (1985).\n \n \n [26] A.F. Io\175400e and A.R. Regel, Prog. Semicond. 4, 237 (1960).\n \n \n [27] R. Dell\20031Anna et al., Phys. Rev. Lett. 80, 1236 (1998).\n \n \n [28] D. Fioretto et al., Phys. Rev. E 59, 4470 (1999).\n \n \n [29] U. Buchenau et al., Phys. Rev. Lett. 77, 4035 (1996).\n \n \n [2] M. Rothenfusser et al., Phys. Rev. B 27, 5196 (1983).\n \n \n [30] J. Mattsson et al., J. Phys.: Condens. Matter 15, S1259 (2003).\n \n \n [31] T. Scopigno et al., Phys. Rev. Lett. 92, 025503 (2004).\n \n \n [32] M. Foret et al., Phys. Rev. Lett. 81, 2100 (1998).\n \n \n [33] F. Sette et al., Science 280, 1550 (1998).\n \n \n [34] J. Wuttke et al., Phys. Rev. E 52, 4026 (1995).\n \n \n [35] M.A. Ramos et al., Phys. Rev. Lett. 78, 82 (1997).\n \n \n [36] G. Monaco et al., Phys. Rev. Lett. 80, 2161 (1998).\n \n \n [37] A. T\250olle, Rep. Prog. Phys. 64, 1473 (2001).\n \n \n [38] As the straight lines do not cross the origin, this does not 2 imply \1623 \21035 \1651 .\n \n \n [39] A. Matic et al., Europhys. Lett. 54, 77 (2001).\n \n \n [3] S. Hunklinger and W. Arnold, in Physical Acoustics, Vol. XII, W.P. Mason and R.N. Thurston Eds. (Academic Press, N.Y. 1976), p. 155.\n \n \n [40] IXS data are usually not available below \1651co, mostly for experimental reasons. E.g., that the rapid onset was not evidenced in vitreous silica [27], is not indicative of its absence but rather of a low qco \21074 1 nm\210221.\n \n \n [41] G. Ruocco et al., Phys. Rev. Lett. 83, 5583 (1999).\n \n \n [42] D. C\1307 iplys et al., J. Physique (Paris) 42, C6-184 (1981).\n \n \n [43] R. Vacher et al., Rev. Sci. Instrum. 51, 288 (1980).\n \n \n [44] R. Vacher et al., arXiv:cond-mat/0505560, 23 May 2005.\n \n \n [45] T.N. Claytor et al., Phys. Rev. B 18, 5842 (1978).\n \n \n [46] M. Arai et al., Physica B 263-264, 268 (1999).\n \n \n [4] R. Vacher et al., J. Non-Cryst. Solids 45, 397 (1981); T.C. Zhu et al., Phys. Rev. B 44, 4281 (1991).\n \n \n [5] J.E. Graebner et al., Phys. Rev. B 34, 5696 (1986).\n \n \n [6] E. Duval and A. Mermet, Phys. Rev. B 58, 8159 (1998).\n \n \n [7] A. Matic et al., Phys. Rev. Lett. 93, 145502 (2004).\n \n \n [8] Often alluded to, e.g. in the Encyclopedia of Materials: Science and Technology, K.H.J. Buschow et al., Eds., Vol. 1 (Elsevier, Oxford, 2001), articles by S.R. Elliott on pp. 171-174 and U. Buchenau on pp. 212-215.\n \n \n [9] E. Rat et al., Phys. Rev. Lett. 83, 1355 (1999).\n \n"; + + public static final String STATISTICS_JSON = + "[{ \"citationsPerYear\": \"many\", \"anotherCoolStatistic\": \"WoW\", \"nestedStat\": { \"firstNestedStat\" : \"value 1\", \"secondNestedStat\" : \"value 2\"}, \"listingStat\" : [ \"one\", \"two\" ] }]"; + + public static Builder getStructuredproperty(final String value, final String classname, final String schemename) { + return getStructuredproperty(value, classname, schemename, null); + } + + public static Builder getStructuredproperty(final String value, final String classname, final String schemename, final DataInfo dataInfo) { + final Builder sp = StructuredProperty.newBuilder().setValue(value).setQualifier(getQualifier(classname, schemename)); + if (dataInfo != null) { + sp.setDataInfo(dataInfo); + } + return sp; + } + + public static Qualifier.Builder getQualifier(final String classname, final String schemename) { + return Qualifier.newBuilder().setClassid(classname).setClassname(classname).setSchemeid(schemename).setSchemename(schemename); + } + + public static KeyValue getKV(final String id, final String name) { + return KeyValue.newBuilder().setKey(id).setValue(name).build(); + } + + public static OafEntity getDatasource(final String datasourceId) { + return OafEntity + .newBuilder() + .setType(Type.datasource) + .setId(datasourceId) + .setDatasource( + Datasource.newBuilder().setMetadata( + Datasource.Metadata.newBuilder().setOfficialname(sf("officialname")).setEnglishname(sf("englishname")) + .setWebsiteurl(sf("websiteurl")).setContactemail(sf("contactemail")).addAccessinfopackage(sf("accessinforpackage")) + .setNamespaceprefix(sf("namespaceprofix")).setDescription(sf("description")).setOdnumberofitems(sf("numberofitems")) + .setOdnumberofitemsdate(sf("numberofitems date")) + // .addOdsubjects("subjects") + .setOdpolicies(sf("policies")).addOdlanguages(sf("languages")).addOdcontenttypes(sf("contenttypes")) + .setDatasourcetype(getQualifier("type class", "type scheme")))).build(); + } + + public static OafEntity getResult(final String id) { + return getResultBuilder(id).build(); + } + + public static OafEntity.Builder getResultBuilder(final String id) { + return OafEntity + .newBuilder() + .setType(Type.result) + .setId(id) + .setResult( + Result.newBuilder() + .setMetadata( + Result.Metadata + .newBuilder() + .addTitle( + getStructuredproperty( + "Analysis of cell viability in intervertebral disc: Effect of endplate permeability on cell population", + "main title", "dnet:result_titles", getDataInfo())) + .addTitle(getStructuredproperty("Another title", "alternative title", "dnet:result_titles", getDataInfo())) + .addSubject(getStructuredproperty("Biophysics", "subject", "dnet:result_sujects")) + .setDateofacceptance(sf("2010-01-01")).addSource(sf("sourceA")).addSource(sf("sourceB")) + .addContext(Context.newBuilder().setId("egi::virtual::970")) + .addContext(Context.newBuilder().setId("egi::classification::natsc::math::applied")) + .addContext(Context.newBuilder().setId("egi::classification::natsc::math")) + .addContext(Context.newBuilder().setId("egi::classification::natsc")) + .addContext(Context.newBuilder().setId("egi::classification")).addContext(Context.newBuilder().setId("egi")) + .addDescription(sf("Responsible for making and maintaining the extracellular matrix ...")) + .addDescription(sf("Another description ...")).setPublisher(sf("ELSEVIER SCI LTD")) + .setResulttype(getQualifier("publication", "dnet:result_types")) + .setLanguage(getQualifier("eng", "dnet:languages"))).addInstance(getInstance("10|od__10", "Uk pubmed")) + .addInstance(getInstance("10|od__10", "arxiv"))) + .addCollectedfrom(getKV("opendoar____::1064", "Oxford University Research Archive")) + .addPid(getStructuredproperty("doi:74293", "doi", "dnet:pids")).addPid(getStructuredproperty("oai:74295", "oai", "dnet:pids")) + .setDateofcollection(""); + } + + public static DataInfo getDataInfo() { + return getDataInfo("0.4"); + } + + public static DataInfo getDataInfo(final String trust) { + return DataInfo.newBuilder().setDeletedbyinference(false).setTrust("0.4").setInferenceprovenance("algo").setProvenanceaction(getQualifier("xx", "yy")) + .build(); + } + + public static Instance.Builder getInstance(final String hostedbyId, final String hostedbyName) { + return Instance.newBuilder().setHostedby(getKV(hostedbyId, hostedbyName)).setAccessright(getQualifier("OpenAccess", "dnet:access_modes")) + .setInstancetype(getQualifier("publication", "dnet:result_typologies")).addUrl("webresource url"); + + } + + public static OafRel getDedupRel(final String source, final String target, final RelType relType, final String relClass) { + return OafRel.newBuilder().setSource(source).setTarget(target).setRelType(relType).setSubRelType(SubRelType.dedup).setRelClass(relClass) + .setChild(false).setCachedTarget(getResult(target)) + .setResultResult(ResultResult.newBuilder().setDedup(Dedup.newBuilder().setRelMetadata(RelMetadata.getDefaultInstance()))) + .build(); + } + + public static OafRel getProjectOrganization(final String source, final String target, final String relClass) throws InvalidProtocolBufferException { + final OafRel.Builder oafRel = OafRel + .newBuilder() + .setSource(source) + .setTarget(target) + .setRelType(RelType.projectOrganization) + .setSubRelType(SubRelType.participation) + .setRelClass(relClass) + .setChild(false) + .setProjectOrganization( + ProjectOrganization.newBuilder().setParticipation( + Participation.newBuilder().setParticipantnumber("" + 1) + .setRelMetadata(relMetadata(relClass, "dnet:project_organization_relations")))); + switch (Participation.RelName.valueOf(relClass)) { + case hasParticipant: + oafRel.setCachedTarget(getProjectFP7(target, "SP3")); + break; + case isParticipant: + oafRel.setCachedTarget(getOrganization(target)); + break; + default: + break; + } + return oafRel.build(); + } + + public static GeneratedMessage getOrganizationOrganization(final String source, final String target, final String relClass) { + final OafRel.Builder oafRel = OafRel + .newBuilder() + .setSource(source) + .setTarget(target) + .setRelType(RelType.organizationOrganization) + .setSubRelType(SubRelType.dedup) + .setRelClass(relClass) + .setChild(true) + .setOrganizationOrganization( + OrganizationOrganization.newBuilder().setDedup( + Dedup.newBuilder().setRelMetadata(relMetadata(relClass, "dnet:organization_organization_relations")))); + + switch (Dedup.RelName.valueOf(relClass)) { + case isMergedIn: + oafRel.setCachedTarget(getOrganization(source)); + break; + case merges: + oafRel.setCachedTarget(getOrganization(target)); + break; + default: + break; + } + return oafRel.build(); + } + + public static OafRel getDatasourceOrganization(final String source, final String target, final String relClass) throws InvalidProtocolBufferException { + final OafRel.Builder oafRel = OafRel + .newBuilder() + .setSource(source) + .setTarget(target) + .setRelType(RelType.datasourceOrganization) + .setSubRelType(SubRelType.provision) + .setRelClass(relClass) + .setChild(false) + .setDatasourceOrganization( + DatasourceOrganization.newBuilder().setProvision( + Provision.newBuilder().setRelMetadata(relMetadata(relClass, "dnet:datasource_organization_relations")))); + switch (Provision.RelName.valueOf(relClass)) { + case isProvidedBy: + oafRel.setCachedTarget(getOrganization(target)); + break; + case provides: + oafRel.setCachedTarget(getDatasource(target)); + break; + default: + break; + } + return oafRel.build(); + } + + public static OafRel getSimilarityRel(final String sourceId, final String targetId, final OafEntity result, final String relClass) { + return OafRel + .newBuilder() + .setSource(sourceId) + .setTarget(targetId) + .setRelType(RelType.resultResult) + .setSubRelType(SubRelType.similarity) + .setRelClass(relClass) + .setChild(false) + .setCachedTarget(result) + .setResultResult( + ResultResult.newBuilder().setSimilarity( + Similarity.newBuilder().setRelMetadata(relMetadata(relClass, "dnet:resultResult_relations")).setSimilarity(.4f) + .setType(Similarity.Type.STANDARD))).build(); + } + + public static RelMetadata.Builder relMetadata(final String classname, final String schemename) { + return RelMetadata.newBuilder().setSemantics(getQualifier(classname, schemename)); + } + + public static OafEntity getOrganization(final String orgId) { + return OafEntity + .newBuilder() + .setType(Type.organization) + .setId(orgId) + .addCollectedfrom(getKV("opendoar_1234", "UK pubmed")) + .setOrganization( + Organization.newBuilder().setMetadata( + Organization.Metadata.newBuilder().setLegalname(sf("CENTRE D'APPUI A LA RECHERCHE ET A LA FORMATION GIE")) + .setLegalshortname(sf("CAREF")).setWebsiteurl(sf("www.caref-mali.org")) + .setCountry(getQualifier("ML", "dnet:countries")))).build(); + } + + public static OafRel getResultProject(final String from, final String to, final OafEntity project, final String relClass) + throws InvalidProtocolBufferException { + return OafRel + .newBuilder() + .setSource(from) + .setTarget(to) + .setRelType(RelType.resultProject) + .setSubRelType(SubRelType.outcome) + .setRelClass(relClass) + .setChild(false) + .setResultProject( + ResultProject.newBuilder().setOutcome(Outcome.newBuilder().setRelMetadata(relMetadata(relClass, "dnet:result_project_relations")))) + .setCachedTarget(project).build(); + } + + public static OafEntity getProjectFP7(final String projectId, final String fundingProgram) throws InvalidProtocolBufferException { + return OafEntity + .newBuilder() + .setType(Type.project) + .setId(projectId) + .addCollectedfrom(getKV("opendoar_1234", "UK pubmed")) + .setProject( + Project.newBuilder() + .setMetadata( + Project.Metadata + .newBuilder() + .setAcronym(sf("5CYRQOL")) + .setTitle(sf("Cypriot Researchers Contribute to our Quality of Life")) + .setStartdate(sf("2007-05-01")) + .setEnddate(sf("2007-10-31")) + .setEcsc39(sf("false")) + .setContracttype(getQualifier("CSA", "ec:FP7contractTypes")) + .addFundingtree( + sf("ec__________::ECECEuropean Commissionec__________::EC::FP7::" + + fundingProgram + + "::PEOPLEMarie-Curie ActionsPEOPLEec:programec__________::EC::FP7::" + + fundingProgram + + "" + + fundingProgram + + "-People" + + fundingProgram + + "ec:specificprogramec__________::EC::FP7SEVENTH FRAMEWORK PROGRAMMEFP7ec:frameworkprogram")))) + .build(); + } + + public static OafEntity getProjectWT() throws InvalidProtocolBufferException { + return OafEntity + .newBuilder() + .setType(Type.project) + .setId("project|wt::087536") + .addCollectedfrom(getKV("wellcomeTrust", "wellcome trust")) + .setProject( + Project.newBuilder() + .setMetadata( + Project.Metadata + .newBuilder() + .setAcronym(sf("UNKNOWN")) + .setTitle(sf("Research Institute for Infectious Diseases of Poverty (IIDP).")) + .setStartdate(sf("2007-05-01")) + .setEnddate(sf("2007-10-31")) + .setEcsc39(sf("false")) + .setContracttype(getQualifier("UNKNOWN", "wt:contractTypes")) + .addFundingtree( + sf("wt__________::WTWTWellcome Trustwt__________::WT::UNKNOWNUNKNOWNUNKNOWNwt:fundingStream")) + .addFundingtree( + sf("wt__________::WTWTWellcome Trustwt__________::WT::Technology TransferTechnology TransferTechnology Transferwt:fundingStream")))) + .build(); + } + + public static ExtraInfo extraInfo(final String name, final String provenance, final String trust, final String typology, final String value) { + final ExtraInfo.Builder e = ExtraInfo.newBuilder().setName(name).setProvenance(provenance).setTrust(trust).setTypology(typology).setValue(value); + return e.build(); + } + + // public static DocumentClasses documentClasses() { + // DocumentClasses.Builder builder = DocumentClasses.newBuilder(); + // for (int i = 0; i < RandomUtils.nextInt(N_DOCUMENT_CLASSES) + 1; i++) { + // builder.addArXivClasses(getDocumentClass()).addDdcClasses(getDocumentClass()).addWosClasses(getDocumentClass()) + // .addMeshEuroPMCClasses(getDocumentClass()); + // } + // return builder.build(); + // } + // + // private static DocumentClass getDocumentClass() { + // DocumentClass.Builder builder = DocumentClass.newBuilder(); + // for (int i = 0; i < RandomUtils.nextInt(N_DOCUMENT_CLASS_LABELS) + 1; i++) { + // builder.addClassLabels("test_class_" + i); + // } + // return builder.setConfidenceLevel(0.5F).build(); + // } + // + // public static DocumentStatistics documentStatistics() { + // return + // DocumentStatistics.newBuilder().setCitationsFromAllPapers(basicCitationStatistics()).setCitationsFromPublishedPapers(basicCitationStatistics()) + // .build(); + // } + // + // private static BasicCitationStatistics basicCitationStatistics() { + // BasicCitationStatistics.Builder builder = BasicCitationStatistics.newBuilder(); + // for (int i = 0; i < N_CITATION_STATS; i++) { + // builder.addNumberOfCitationsPerYear(statisticsKeyValue()); + // builder.setNumberOfCitations(RandomUtils.nextInt(5) + 1); + // } + // return builder.build(); + // } + // + // private static StatisticsKeyValue statisticsKeyValue() { + // return StatisticsKeyValue.newBuilder().setKey((RandomUtils.nextInt(30) + 1980) + "").setValue(RandomUtils.nextInt(5) + 1).build(); + // } + // + // public static AuthorStatistics authorStatistics() { + // AuthorStatistics.Builder builder = AuthorStatistics.newBuilder(); + // builder.setCore(commonCoreStatistics()); + // for (int i = 0; i < N_COAUTHORS; i++) { + // builder.addCoAuthors(coAuthor()); + // } + // return builder.build(); + // } + // + // private static CoAuthor coAuthor() { + // CoAuthor.Builder builder = CoAuthor.newBuilder(); + // builder.setId("30|od______2345::" + Hashing.md5(RandomStringUtils.random(10))); + // builder.setCoauthoredPapersCount(RandomUtils.nextInt(5) + 1); + // return builder.build(); + // } + // + // public static CommonCoreStatistics commonCoreStatistics() { + // CommonCoreStatistics.Builder builder = CommonCoreStatistics.newBuilder(); + // + // builder.setAllPapers(coreStatistics()); + // builder.setPublishedPapers(coreStatistics()); + // + // return builder.build(); + // } + // + // private static CoreStatistics coreStatistics() { + // CoreStatistics.Builder builder = CoreStatistics.newBuilder(); + // + // builder.setNumberOfPapers(RandomUtils.nextInt(10)); + // builder.setCitationsFromAllPapers(extendedStatistics()); + // builder.setCitationsFromPublishedPapers(extendedStatistics()); + // + // return builder.build(); + // } + // + // private static ExtendedStatistics extendedStatistics() { + // ExtendedStatistics.Builder builder = ExtendedStatistics.newBuilder(); + // + // builder.setBasic(basicCitationStatistics()); + // builder.setAverageNumberOfCitationsPerPaper(RandomUtils.nextFloat()); + // for (int i = 0; i < N_CITATION_STATS; i++) { + // builder.addNumberOfPapersCitedAtLeastXTimes(statisticsKeyValue()); + // } + // + // return builder.build(); + // } + + public static StringField sf(final String s) { + return sf(s, null); + } + + public static StringField sf(final String s, final DataInfo dataInfo) { + final StringField.Builder sf = StringField.newBuilder().setValue(s); + if (dataInfo != null) { + sf.setDataInfo(dataInfo); + } + return sf.build(); + } + + public static OafDecoder embed(final GeneratedMessage msg, + final Kind kind, + final boolean deletedByInference, + final boolean inferred, + final String provenance, + final String action) { + + final Oaf.Builder oaf = Oaf + .newBuilder() + .setKind(kind) + .setLastupdatetimestamp(System.currentTimeMillis()) + .setDataInfo( + DataInfo.newBuilder().setDeletedbyinference(deletedByInference).setInferred(inferred).setTrust("0.5") + .setInferenceprovenance(provenance).setProvenanceaction(getQualifier(action, action))); + switch (kind) { + case entity: + oaf.setEntity((OafEntity) msg); + break; + case relation: + oaf.setRel((OafRel) msg); + break; + default: + break; + } + + return OafDecoder.decode(oaf.build()); + } + + public static OafDecoder embed(final GeneratedMessage msg, final Kind kind) { + return embed(msg, kind, false, false, "inference_provenance", "provenance_action"); + } + +} diff --git a/dnet-dedup-test/src/test/java/eu/dnetlib/pace/distance/DetectorTest.java b/dnet-dedup-test/src/test/java/eu/dnetlib/pace/distance/DetectorTest.java index 90177e8..b886df7 100644 --- a/dnet-dedup-test/src/test/java/eu/dnetlib/pace/distance/DetectorTest.java +++ b/dnet-dedup-test/src/test/java/eu/dnetlib/pace/distance/DetectorTest.java @@ -1,450 +1,360 @@ -//package eu.dnetlib.pace.distance; -// -//import com.google.common.collect.Lists; -//import com.google.common.collect.Maps; -//import com.google.common.collect.Sets; -//import com.googlecode.protobuf.format.JsonFormat; -//import eu.dnetlib.data.proto.OafProtos; -//import eu.dnetlib.pace.AbstractProtoPaceTest; -//import eu.dnetlib.pace.clustering.BlacklistAwareClusteringCombiner; -//import eu.dnetlib.pace.config.Config; -//import eu.dnetlib.pace.config.DedupConfig; -//import eu.dnetlib.pace.distance.eval.ScoreResult; -//import eu.dnetlib.pace.model.MapDocument; -//import eu.dnetlib.pace.model.ProtoDocumentBuilder; -//import org.apache.commons.io.IOUtils; -//import org.apache.commons.logging.Log; -//import org.apache.commons.logging.LogFactory; -//import org.junit.Ignore; -//import org.junit.Test; -// -//import java.io.IOException; -//import java.util.List; -//import java.util.Map; -//import java.util.Set; -//import java.util.stream.Collectors; -// -//import static org.junit.Assert.assertFalse; -//import static org.junit.Assert.assertTrue; -// -//public class DetectorTest extends AbstractProtoPaceTest { -// -// private static final Log log = LogFactory.getLog(DetectorTest.class); -// -// @Test -// public void testDistanceResultSimple() { -// final Config config = getResultSimpleConf(); -// -// final MapDocument resA = result(config, "A", "Recent results from CDF"); -// final MapDocument resB = result(config, "B", "Recent results from CDF"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// final double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue(d == 1.0); -// } -// -// @Test -// public void testDistanceResultSimpleMissingDates() { -// final Config config = getResultSimpleConf(); -// -// final MapDocument resA = result(config, "A", "Recent results from BES"); -// final MapDocument resB = result(config, "A", "Recent results from CES"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// final double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue(d > 0.97); -// } -// -// @Test -// public void testDistanceResultInvalidDate() { -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "title title title 6BESR", "2013-01-05"); -// final MapDocument resB = result(config, "B", "title title title 6BESR", "qwerty"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// final double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue(d == 1.0); -// } -// -// @Ignore -// @Test -// public void testDistanceResultMissingOneDate() { -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "title title title 6BESR", null); -// final MapDocument resB = result(config, "B", "title title title 6CLER", "2012-02"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue((d > 0.9) && (d < 1.0)); -// } -// -// @Ignore -// @Test -// public void testDistanceResult() { -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "title title title BES", ""); -// final MapDocument resB = result(config, "B", "title title title CLEO"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue((d > 0.9) && (d < 1.0)); -// } -// -// @Ignore -// @Test -// public void testDistanceResultMissingTwoDate() { -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "bellaciao"); -// final MapDocument resB = result(config, "B", "bellocioa"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue((d > 0.9) && (d < 1.0)); -// } -// -// @Ignore -// @Test -// public void testDistanceOrganizationIgnoreMissing() { -// -// final Config config = getOrganizationSimpleConf(); -// -// final MapDocument orgA = organization(config, "A", "CONSIGLIO NAZIONALE DELLE RICERCHE"); -// final MapDocument orgB = organization(config, "B", "CONSIGLIO NAZIONALE DELLE RICERCHE", "CNR"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(orgA, orgB, config); -// final double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue(d > 0.99); -// } -// -// @Test -// public void testDistanceResultCase1() { -// -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003"); -// final MapDocument resB = result(config, "B", "Search for the Standard Model Higgs Boson", "2003"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue((d > 0.9) && (d < 1.0)); -// } -// -// @Test -// public void testDistanceResultCaseDoiMatch1() { -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003", "10.1594/PANGAEA.726855"); -// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", "10.1594/PANGAEA.726855"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue("exact DOIs will produce an exact match", d == 1.0); -// } -// -// @Test -// public void testDistanceResultCaseDoiMatch2() { -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1594/PANGAEA.726855"); -// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2005", "10.1594/PANGAEA.726855"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue("exact DOIs will produce an exact match, regardless of different titles or publication years", d == 1.0); -// } -// -// @Test -// public void testDistanceResultCaseDoiMatch3() { -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024"); -// final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue("a missing DOI will casue the comparsion to continue with the following conditions", d == 1.0); -// } -// -// @Test -// public void testDistanceResultCaseDoiMatch4() { -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024"); -// final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2005"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue("a missing DOI, comparsion continues with the following conditions, different publication years will drop the score to 0", d == 0.0); -// } -// -// @Test -// public void testDistanceResultCaseDoiMatch5() { -// -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "Search for the Standard Model Higgs Boson", "2003", "10.1016/j.jmb.2010.12.020"); -// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue("a missing DOI, comparsion continues with the following conditions", (d > 0.9) && (d < 1.0)); -// } -// -// @Test -// public void testDistanceResultCaseDoiMatch6() { -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024"); -// final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003", "anotherDifferentDOI"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue("different DOIs will NOT drop the score to 0, then evaluate other fields", d == 1.0); -// } -// -// @Test -// public void testDistanceResultCaseDoiMatch7() { -// final Config config = getResultConf(); -// -// final MapDocument resA = result(config, "A", "Adrenal Insufficiency asd asd", "1951", Lists.newArrayList("PMC2037944", "axdsds")); -// final MapDocument resB = result(config, "B", "Adrenal Insufficiency", "1951", "PMC2037944"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue("different DOIs will drop the score to 0, regardless of the other fields", d > 0.9 & d < 1); -// } -// -// // http://dx.doi.org/10.1594/PANGAEA.726855 doi:10.1594/PANGAEA.726855 -// -// @Test -// public void testDistanceResultCaseAuthor1() { -// -// final Config config = getResultAuthorsConf(); -// -// final List authorsA = Lists.newArrayList("a", "b", "c", "d"); -// final List authorsB = Lists.newArrayList("a", "b", "c"); -// final List pid = Lists.newArrayList(); -// -// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); -// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// final double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue(d == 0.0); -// } -// -// @Test -// public void testDistanceResultCaseAuthor2() { -// -// final Config config = getResultAuthorsConf(); -// -// final List authorsA = Lists.newArrayList("a", "b", "c"); -// final List authorsB = Lists.newArrayList("a", "b", "c"); -// final List pid = Lists.newArrayList(); -// -// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); -// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// final double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue(d == 1.0); -// } -// -// @Test -// public void testDistanceResultCaseAuthor3() { -// -// final Config config = getResultAuthorsConf(); -// -// final List authorsA = Lists.newArrayList("Bardi, A.", "Manghi, P.", "Artini, M."); -// final List authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele"); -// final List pid = Lists.newArrayList(); -// -// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); -// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// assertTrue((d > 0.9) && (d < 1.0)); -// } -// -// @Test -// public void testDistanceResultCaseAuthor4() { -// -// final Config config = getResultAuthorsConf(); -// -// final List authorsA = Lists.newArrayList("Bardi, Alessia", "Manghi, Paolo", "Artini, Michele", "a"); -// final List authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele"); -// final List pid = Lists.newArrayList(); -// -// final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); -// final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// final double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// // assertTrue(d.getScore() == 0.0); -// } -// -// @Test -// public void testDistanceResultNoPidsConf() { -// -// final Config config = getResultFullConf(); -// -// final MapDocument resA = -// result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010"); -// -// final MapDocument resB = -// result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reportsX", "2010"); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// final double s = sr.getScore(); -// -// log.info(sr.toString()); -// log.info(String.format(" s ---> %s", s)); -// // assertTrue(d.getScore() == 0.0); -// } -// -// @Test -// public void testDistanceResultPidsConf() { -// -// final Config config = getResultFullConf(); -// -// final List authorsA = Lists.newArrayList("Nagarajan Pranesh", "Guy Vautier", "Punyanganie de Silva"); -// final List authorsB = Lists.newArrayList("Pranesh Nagarajan", "Vautier Guy", "de Silva Punyanganie"); -// -// final List pidA = Lists.newArrayList("10.1186/1752-1947-4-299", "a", "b"); -// final MapDocument resA = -// result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010", -// pidA, authorsA); -// -// final List pidB = Lists.newArrayList("c", "a", "10.1186/1752-1947-4-299", "d"); -// final MapDocument resB = -// result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reportsX", "2010", -// pidB, authorsB); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// final double s = sr.getScore(); -// log.info(sr.toString()); -// log.info(String.format(" s ---> %s", s)); -// -// // assertTrue(d.getScore() == 0.0); -// } -// -// @Test -// public void testDistanceResultFullConf() { -// -// final Config config = getResultFullConf(); -// -// final List authorsA = Lists.newArrayList("Nagarajan Pranesh", "Guy Vautier", "Punyanganie de Silva"); -// final List authorsB = Lists.newArrayList("Pranesh Nagarajan", "Vautier Guy", "de Silva Punyanganie"); -// -// final MapDocument resA = -// result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010", -// "10.1186/1752-1947-4-299", authorsA); -// -// final MapDocument resB = -// result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010", -// "10.1186/1752-1947-4-299", authorsB); -// -// final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); -// final double d = sr.getScore(); -// log.info(String.format(" d ---> %s", d)); -// -// // assertTrue(d.getScore() == 0.0); -// } -// -// @Ignore -// @Test -// public void testDistance() throws IOException { -// -// final DedupConfig conf = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.prod.pace.json")); -// -// final MapDocument crossref = asMapDocument(conf, "/eu/dnetlib/pace/crossref.json"); -// final MapDocument alicante = asMapDocument(conf, "/eu/dnetlib/pace/alicante.json"); -// -// final ScoreResult result = new PaceDocumentDistance().between(crossref, alicante, conf); -// -// log.info("score = " + result); -// -// } -// -// @Ignore -// @Test -// public void testDistanceOrgs() throws IOException { -// -// final DedupConfig conf = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf")); -// -// final MapDocument orgA = asMapDocument(conf, readFromClasspath("/eu/dnetlib/pace/organization1.json")); -// final MapDocument orgB = asMapDocument(conf, readFromClasspath("/eu/dnetlib/pace/organization2.json")); -// -// Set keysA = getGroupingKeys(conf, orgA); -// Set keysB = getGroupingKeys(conf, orgB); -// -// assertFalse(String.format("A: %s\nB: %s", keysA, keysB), Sets.intersection(keysA, keysB).isEmpty()); -// -// log.info("clustering keys A = " + getGroupingKeys(conf, orgA)); -// log.info("clustering keys B = " + getGroupingKeys(conf, orgB)); -// -// final ScoreResult result = new PaceDocumentDistance().between(orgA, orgB, conf); -// -// log.info("score = " + result); -// log.info("distance = " + result.getScore()); -// } -// -// private Set getGroupingKeys(DedupConfig conf, MapDocument doc) { -// return Sets.newHashSet(BlacklistAwareClusteringCombiner.filterAndCombine(doc, conf)); -// } -// -// private MapDocument asMapDocument(DedupConfig conf, final String json) { -// OafProtos.OafEntity.Builder b = OafProtos.OafEntity.newBuilder(); -// try { -// JsonFormat.merge(json, b); -// } catch (JsonFormat.ParseException e) { -// throw new IllegalArgumentException(e); -// } -// return ProtoDocumentBuilder.newInstance(b.getId(), b.build(), conf.getPace().getModel()); -// } -// -// -//} +package eu.dnetlib.pace.distance; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.googlecode.protobuf.format.JsonFormat; +import eu.dnetlib.data.proto.OafProtos; +import eu.dnetlib.pace.AbstractProtoPaceTest; +import eu.dnetlib.pace.clustering.BlacklistAwareClusteringCombiner; +import eu.dnetlib.pace.config.Config; +import eu.dnetlib.pace.config.DedupConfig; +import eu.dnetlib.pace.distance.eval.ScoreResult; +import eu.dnetlib.pace.model.MapDocument; +import eu.dnetlib.pace.model.ProtoDocumentBuilder; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class DetectorTest extends AbstractProtoPaceTest { + + private static final Log log = LogFactory.getLog(DetectorTest.class); + + @Test + public void testDistanceResultSimple() { + final Config config = getResultSimpleConf(); + final MapDocument resA = result(config, "A", "Recent results from CDF"); + final MapDocument resB = result(config, "B", "Recent results from CDF"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + final double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue(d == 1.0); + } + + @Test + public void testDistanceResultSimpleMissingDates() { + final Config config = getResultSimpleConf(); + final MapDocument resA = result(config, "A", "Recent results from BES"); + final MapDocument resB = result(config, "A", "Recent results from CES"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + final double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue(d > 0.97); + } + + @Test + public void testDistanceResultInvalidDate() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "title title title 6BESR", "2013-01-05"); + final MapDocument resB = result(config, "B", "title title title 6BESR", "qwerty"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + final double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue(d == 1.0); + } + + @Ignore + @Test + public void testDistanceResultMissingOneDate() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "title title title 6BESR", null); + final MapDocument resB = result(config, "B", "title title title 6CLER", "2012-02"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue((d > 0.9) && (d < 1.0)); + } + + @Ignore + @Test + public void testDistanceResult() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "title title title BES", ""); + final MapDocument resB = result(config, "B", "title title title CLEO"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue((d > 0.9) && (d < 1.0)); + } + + @Ignore + @Test + public void testDistanceResultMissingTwoDate() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "bellaciao"); + final MapDocument resB = result(config, "B", "bellocioa"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue((d > 0.9) && (d < 1.0)); + } + + @Ignore + @Test + public void testDistanceOrganizationIgnoreMissing() { + final Config config = getOrganizationSimpleConf(); + final MapDocument orgA = organization(config, "A", "CONSIGLIO NAZIONALE DELLE RICERCHE"); + final MapDocument orgB = organization(config, "B", "CONSIGLIO NAZIONALE DELLE RICERCHE", "CNR"); + final ScoreResult sr = new PaceDocumentDistance().between(orgA, orgB, config); + final double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue(d > 0.99); + } + + + @Test + public void testDistanceOrganizations() { + final Config config = getOrganizationTestConf(); + final MapDocument orgA = organization(config, "A", "UNIVERSITA DEGLI STUDI DI VERONA"); + final MapDocument orgB = organization(config, "B", "UNIVERSITY OF GENOVA"); + final ScoreResult sr = new PaceDocumentDistance().between(orgA, orgB, config); + final double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + } + + + @Test + public void testDistanceResultCase1() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003"); + final MapDocument resB = result(config, "B", "Search for the Standard Model Higgs Boson", "2003"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue((d > 0.9) && (d < 1.0)); + } + + @Test + public void testDistanceResultCaseDoiMatch1() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003", "10.1594/PANGAEA.726855"); + final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", "10.1594/PANGAEA.726855"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue("exact DOIs will produce an exact match", d == 1.0); + } + + @Test + public void testDistanceResultCaseDoiMatch2() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1594/PANGAEA.726855"); + final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2005", "10.1594/PANGAEA.726855"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue("exact DOIs will produce an exact match, regardless of different titles or publication years", d == 1.0); + } + + @Test + public void testDistanceResultCaseDoiMatch3() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024"); + final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue("a missing DOI will casue the comparsion to continue with the following conditions", d == 1.0); + } + + @Test + public void testDistanceResultCaseDoiMatch4() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024"); + final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2005"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue("a missing DOI, comparsion continues with the following conditions, different publication years will drop the score to 0", d == 0.0); + } + + @Test + public void testDistanceResultCaseDoiMatch5() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "Search for the Standard Model Higgs Boson", "2003", "10.1016/j.jmb.2010.12.020"); + final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue("a missing DOI, comparsion continues with the following conditions", (d > 0.9) && (d < 1.0)); + } + + @Test + public void testDistanceResultCaseDoiMatch6() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024"); + final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003", "anotherDifferentDOI"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue("different DOIs will NOT drop the score to 0, then evaluate other fields", d == 1.0); + } + + @Test + public void testDistanceResultCaseDoiMatch7() { + final Config config = getResultConf(); + final MapDocument resA = result(config, "A", "Adrenal Insufficiency asd asd", "1951", Lists.newArrayList("PMC2037944", "axdsds")); + final MapDocument resB = result(config, "B", "Adrenal Insufficiency", "1951", "PMC2037944"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue("different DOIs will drop the score to 0, regardless of the other fields", d > 0.9 & d < 1); + } + + // http://dx.doi.org/10.1594/PANGAEA.726855 doi:10.1594/PANGAEA.726855 + @Test + public void testDistanceResultCaseAuthor1() { + final Config config = getResultAuthorsConf(); + final List authorsA = Lists.newArrayList("a", "b", "c", "d"); + final List authorsB = Lists.newArrayList("a", "b", "c"); + final List pid = Lists.newArrayList(); + final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); + final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + final double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue(d == 0.0); + } + + @Test + public void testDistanceResultCaseAuthor2() { + final Config config = getResultAuthorsConf(); + final List authorsA = Lists.newArrayList("a", "b", "c"); + final List authorsB = Lists.newArrayList("a", "b", "c"); + final List pid = Lists.newArrayList(); + final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); + final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + final double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue(d == 1.0); + } + + @Test + public void testDistanceResultCaseAuthor3() { + final Config config = getResultAuthorsConf(); + final List authorsA = Lists.newArrayList("Bardi, A.", "Manghi, P.", "Artini, M."); + final List authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele"); + final List pid = Lists.newArrayList(); + final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); + final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + assertTrue((d > 0.9) && (d < 1.0)); + } + + @Test + public void testDistanceResultCaseAuthor4() { + final Config config = getResultAuthorsConf(); + final List authorsA = Lists.newArrayList("Bardi, Alessia", "Manghi, Paolo", "Artini, Michele", "a"); + final List authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele"); + final List pid = Lists.newArrayList(); + final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA); + final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + final double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + // assertTrue(d.getScore() == 0.0); + } + + @Test + public void testDistanceResultNoPidsConf() { + final Config config = getResultFullConf(); + final MapDocument resA = + result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010"); + final MapDocument resB = + result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reportsX", "2010"); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + final double s = sr.getScore(); + log.info(sr.toString()); + log.info(String.format(" s ---> %s", s)); + // assertTrue(d.getScore() == 0.0); + } + + @Test + public void testDistanceResultPidsConf() { + final Config config = getResultFullConf(); + final List authorsA = Lists.newArrayList("Nagarajan Pranesh", "Guy Vautier", "Punyanganie de Silva"); + final List authorsB = Lists.newArrayList("Pranesh Nagarajan", "Vautier Guy", "de Silva Punyanganie"); + final List pidA = Lists.newArrayList("10.1186/1752-1947-4-299", "a", "b"); + final MapDocument resA = + result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010", + pidA, authorsA); + final List pidB = Lists.newArrayList("c", "a", "10.1186/1752-1947-4-299", "d"); + final MapDocument resB = + result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reportsX", "2010", + pidB, authorsB); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + final double s = sr.getScore(); + log.info(sr.toString()); + log.info(String.format(" s ---> %s", s)); + // assertTrue(d.getScore() == 0.0); + } + + @Test + public void testDistanceResultFullConf() { + final Config config = getResultFullConf(); + final List authorsA = Lists.newArrayList("Nagarajan Pranesh", "Guy Vautier", "Punyanganie de Silva"); + final List authorsB = Lists.newArrayList("Pranesh Nagarajan", "Vautier Guy", "de Silva Punyanganie"); + final MapDocument resA = + result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010", + "10.1186/1752-1947-4-299", authorsA); + final MapDocument resB = + result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010", + "10.1186/1752-1947-4-299", authorsB); + final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config); + final double d = sr.getScore(); + log.info(String.format(" d ---> %s", d)); + // assertTrue(d.getScore() == 0.0); + } + + @Ignore + @Test + public void testDistance() throws IOException { + final DedupConfig conf = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.prod.pace.json")); + final MapDocument crossref = asMapDocument(conf, "/eu/dnetlib/pace/crossref.json"); + final MapDocument alicante = asMapDocument(conf, "/eu/dnetlib/pace/alicante.json"); + final ScoreResult result = new PaceDocumentDistance().between(crossref, alicante, conf); + log.info("score = " + result); + } + + @Ignore + @Test + public void testDistanceOrgs() throws IOException { + final DedupConfig conf = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf")); + final MapDocument orgA = asMapDocument(conf, readFromClasspath("/eu/dnetlib/pace/organization1.json")); + final MapDocument orgB = asMapDocument(conf, readFromClasspath("/eu/dnetlib/pace/organization2.json")); + Set keysA = getGroupingKeys(conf, orgA); + Set keysB = getGroupingKeys(conf, orgB); + assertFalse(String.format("A: %s\nB: %s", keysA, keysB), Sets.intersection(keysA, keysB).isEmpty()); + log.info("clustering keys A = " + getGroupingKeys(conf, orgA)); + log.info("clustering keys B = " + getGroupingKeys(conf, orgB)); + final ScoreResult result = new PaceDocumentDistance().between(orgA, orgB, conf); + log.info("score = " + result); + log.info("distance = " + result.getScore()); + } + + private Set getGroupingKeys(DedupConfig conf, MapDocument doc) { + return Sets.newHashSet(BlacklistAwareClusteringCombiner.filterAndCombine(doc, conf)); + } + + private MapDocument asMapDocument(DedupConfig conf, final String json) { + OafProtos.OafEntity.Builder b = OafProtos.OafEntity.newBuilder(); + try { + JsonFormat.merge(json, b); + } catch (JsonFormat.ParseException e) { + throw new IllegalArgumentException(e); + } + return ProtoDocumentBuilder.newInstance(b.getId(), b.build(), conf.getPace().getModel()); + + } +} diff --git a/dnet-dedup-test/src/test/java/eu/dnetlib/pace/distance/DiffPatchMatch.java b/dnet-dedup-test/src/test/java/eu/dnetlib/pace/distance/DiffPatchMatch.java new file mode 100644 index 0000000..0e9283f --- /dev/null +++ b/dnet-dedup-test/src/test/java/eu/dnetlib/pace/distance/DiffPatchMatch.java @@ -0,0 +1,2471 @@ +package eu.dnetlib.pace.distance; + +/* + * Diff Match and Patch + * Copyright 2018 The diff-match-patch Authors. + * https://github.com/google/diff-match-patch + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/* + * Functions for diff, match and patch. + * Computes the difference between two texts to create a patch. + * Applies the patch onto another text, allowing for errors. + * + * @author fraser@google.com (Neil Fraser) + */ + +/** + * Class containing the diff, match and patch methods. + * Also contains the behaviour settings. + */ +public class DiffPatchMatch { + + // Defaults. + // Set these on your diff_match_patch instance to override the defaults. + + /** + * Number of seconds to map a diff before giving up (0 for infinity). + */ + public float Diff_Timeout = 1.0f; + /** + * Cost of an empty edit operation in terms of edit characters. + */ + public short Diff_EditCost = 4; + /** + * At what point is no match declared (0.0 = perfection, 1.0 = very loose). + */ + public float Match_Threshold = 0.5f; + /** + * How far to search for a match (0 = exact location, 1000+ = broad match). + * A match this many characters away from the expected location will add + * 1.0 to the score (0.0 is a perfect match). + */ + public int Match_Distance = 1000; + /** + * When deleting a large block of text (over ~64 characters), how close do + * the contents have to be to match the expected contents. (0.0 = perfection, + * 1.0 = very loose). Note that Match_Threshold controls how closely the + * end points of a delete need to match. + */ + public float Patch_DeleteThreshold = 0.5f; + /** + * Chunk size for context length. + */ + public short Patch_Margin = 4; + + /** + * The number of bits in an int. + */ + private short Match_MaxBits = 32; + + /** + * Internal class for returning results from diff_linesToChars(). + * Other less paranoid languages just use a three-element array. + */ + protected static class LinesToCharsResult { + protected String chars1; + protected String chars2; + protected List lineArray; + + protected LinesToCharsResult(String chars1, String chars2, + List lineArray) { + this.chars1 = chars1; + this.chars2 = chars2; + this.lineArray = lineArray; + } + } + + + // DIFF FUNCTIONS + + + /** + * The data structure representing a diff is a Linked list of Diff objects: + * {Diff(Operation.DELETE, "Hello"), Diff(Operation.INSERT, "Goodbye"), + * Diff(Operation.EQUAL, " world.")} + * which means: delete "Hello", add "Goodbye" and keep " world." + */ + public enum Operation { + DELETE, INSERT, EQUAL + } + + /** + * Find the differences between two texts. + * Run a faster, slightly less optimal diff. + * This method allows the 'checklines' of diff_main() to be optional. + * Most of the time checklines is wanted, so default to true. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @return Linked List of Diff objects. + */ + public LinkedList diff_main(String text1, String text2) { + return diff_main(text1, text2, true); + } + + /** + * Find the differences between two texts. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param checklines Speedup flag. If false, then don't run a + * line-level diff first to identify the changed areas. + * If true, then run a faster slightly less optimal diff. + * @return Linked List of Diff objects. + */ + public LinkedList diff_main(String text1, String text2, + boolean checklines) { + // Set a deadline by which time the diff must be complete. + long deadline; + if (Diff_Timeout <= 0) { + deadline = Long.MAX_VALUE; + } else { + deadline = System.currentTimeMillis() + (long) (Diff_Timeout * 1000); + } + return diff_main(text1, text2, checklines, deadline); + } + + /** + * Find the differences between two texts. Simplifies the problem by + * stripping any common prefix or suffix off the texts before diffing. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param checklines Speedup flag. If false, then don't run a + * line-level diff first to identify the changed areas. + * If true, then run a faster slightly less optimal diff. + * @param deadline Time when the diff should be complete by. Used + * internally for recursive calls. Users should set DiffTimeout instead. + * @return Linked List of Diff objects. + */ + private LinkedList diff_main(String text1, String text2, + boolean checklines, long deadline) { + // Check for null inputs. + if (text1 == null || text2 == null) { + throw new IllegalArgumentException("Null inputs. (diff_main)"); + } + + // Check for equality (speedup). + LinkedList diffs; + if (text1.equals(text2)) { + diffs = new LinkedList(); + if (text1.length() != 0) { + diffs.add(new Diff(Operation.EQUAL, text1)); + } + return diffs; + } + + // Trim off common prefix (speedup). + int commonlength = diff_commonPrefix(text1, text2); + String commonprefix = text1.substring(0, commonlength); + text1 = text1.substring(commonlength); + text2 = text2.substring(commonlength); + + // Trim off common suffix (speedup). + commonlength = diff_commonSuffix(text1, text2); + String commonsuffix = text1.substring(text1.length() - commonlength); + text1 = text1.substring(0, text1.length() - commonlength); + text2 = text2.substring(0, text2.length() - commonlength); + + // Compute the diff on the middle block. + diffs = diff_compute(text1, text2, checklines, deadline); + + // Restore the prefix and suffix. + if (commonprefix.length() != 0) { + diffs.addFirst(new Diff(Operation.EQUAL, commonprefix)); + } + if (commonsuffix.length() != 0) { + diffs.addLast(new Diff(Operation.EQUAL, commonsuffix)); + } + + diff_cleanupMerge(diffs); + return diffs; + } + + /** + * Find the differences between two texts. Assumes that the texts do not + * have any common prefix or suffix. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param checklines Speedup flag. If false, then don't run a + * line-level diff first to identify the changed areas. + * If true, then run a faster slightly less optimal diff. + * @param deadline Time when the diff should be complete by. + * @return Linked List of Diff objects. + */ + private LinkedList diff_compute(String text1, String text2, + boolean checklines, long deadline) { + LinkedList diffs = new LinkedList(); + + if (text1.length() == 0) { + // Just add some text (speedup). + diffs.add(new Diff(Operation.INSERT, text2)); + return diffs; + } + + if (text2.length() == 0) { + // Just delete some text (speedup). + diffs.add(new Diff(Operation.DELETE, text1)); + return diffs; + } + + String longtext = text1.length() > text2.length() ? text1 : text2; + String shorttext = text1.length() > text2.length() ? text2 : text1; + int i = longtext.indexOf(shorttext); + if (i != -1) { + // Shorter text is inside the longer text (speedup). + Operation op = (text1.length() > text2.length()) ? + Operation.DELETE : Operation.INSERT; + diffs.add(new Diff(op, longtext.substring(0, i))); + diffs.add(new Diff(Operation.EQUAL, shorttext)); + diffs.add(new Diff(op, longtext.substring(i + shorttext.length()))); + return diffs; + } + + if (shorttext.length() == 1) { + // Single character string. + // After the previous speedup, the character can't be an equality. + diffs.add(new Diff(Operation.DELETE, text1)); + diffs.add(new Diff(Operation.INSERT, text2)); + return diffs; + } + + // Check to see if the problem can be split in two. + String[] hm = diff_halfMatch(text1, text2); + if (hm != null) { + // A half-match was found, sort out the return data. + String text1_a = hm[0]; + String text1_b = hm[1]; + String text2_a = hm[2]; + String text2_b = hm[3]; + String mid_common = hm[4]; + // Send both pairs off for separate processing. + LinkedList diffs_a = diff_main(text1_a, text2_a, + checklines, deadline); + LinkedList diffs_b = diff_main(text1_b, text2_b, + checklines, deadline); + // Merge the results. + diffs = diffs_a; + diffs.add(new Diff(Operation.EQUAL, mid_common)); + diffs.addAll(diffs_b); + return diffs; + } + + if (checklines && text1.length() > 100 && text2.length() > 100) { + return diff_lineMode(text1, text2, deadline); + } + + return diff_bisect(text1, text2, deadline); + } + + /** + * Do a quick line-level diff on both strings, then rediff the parts for + * greater accuracy. + * This speedup can produce non-minimal diffs. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param deadline Time when the diff should be complete by. + * @return Linked List of Diff objects. + */ + private LinkedList diff_lineMode(String text1, String text2, + long deadline) { + // Scan the text on a line-by-line basis first. + LinesToCharsResult a = diff_linesToChars(text1, text2); + text1 = a.chars1; + text2 = a.chars2; + List linearray = a.lineArray; + + LinkedList diffs = diff_main(text1, text2, false, deadline); + + // Convert the diff back to original text. + diff_charsToLines(diffs, linearray); + // Eliminate freak matches (e.g. blank lines) + diff_cleanupSemantic(diffs); + + // Rediff any replacement blocks, this time character-by-character. + // Add a dummy entry at the end. + diffs.add(new Diff(Operation.EQUAL, "")); + int count_delete = 0; + int count_insert = 0; + String text_delete = ""; + String text_insert = ""; + ListIterator pointer = diffs.listIterator(); + Diff thisDiff = pointer.next(); + while (thisDiff != null) { + switch (thisDiff.operation) { + case INSERT: + count_insert++; + text_insert += thisDiff.text; + break; + case DELETE: + count_delete++; + text_delete += thisDiff.text; + break; + case EQUAL: + // Upon reaching an equality, check for prior redundancies. + if (count_delete >= 1 && count_insert >= 1) { + // Delete the offending records and add the merged ones. + pointer.previous(); + for (int j = 0; j < count_delete + count_insert; j++) { + pointer.previous(); + pointer.remove(); + } + for (Diff subDiff : diff_main(text_delete, text_insert, false, + deadline)) { + pointer.add(subDiff); + } + } + count_insert = 0; + count_delete = 0; + text_delete = ""; + text_insert = ""; + break; + } + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + diffs.removeLast(); // Remove the dummy entry at the end. + + return diffs; + } + + /** + * Find the 'middle snake' of a diff, split the problem in two + * and return the recursively constructed diff. + * See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param deadline Time at which to bail if not yet complete. + * @return LinkedList of Diff objects. + */ + protected LinkedList diff_bisect(String text1, String text2, + long deadline) { + // Cache the text lengths to prevent multiple calls. + int text1_length = text1.length(); + int text2_length = text2.length(); + int max_d = (text1_length + text2_length + 1) / 2; + int v_offset = max_d; + int v_length = 2 * max_d; + int[] v1 = new int[v_length]; + int[] v2 = new int[v_length]; + for (int x = 0; x < v_length; x++) { + v1[x] = -1; + v2[x] = -1; + } + v1[v_offset + 1] = 0; + v2[v_offset + 1] = 0; + int delta = text1_length - text2_length; + // If the total number of characters is odd, then the front path will + // collide with the reverse path. + boolean front = (delta % 2 != 0); + // Offsets for start and end of k loop. + // Prevents mapping of space beyond the grid. + int k1start = 0; + int k1end = 0; + int k2start = 0; + int k2end = 0; + for (int d = 0; d < max_d; d++) { + // Bail out if deadline is reached. + if (System.currentTimeMillis() > deadline) { + break; + } + + // Walk the front path one step. + for (int k1 = -d + k1start; k1 <= d - k1end; k1 += 2) { + int k1_offset = v_offset + k1; + int x1; + if (k1 == -d || (k1 != d && v1[k1_offset - 1] < v1[k1_offset + 1])) { + x1 = v1[k1_offset + 1]; + } else { + x1 = v1[k1_offset - 1] + 1; + } + int y1 = x1 - k1; + while (x1 < text1_length && y1 < text2_length + && text1.charAt(x1) == text2.charAt(y1)) { + x1++; + y1++; + } + v1[k1_offset] = x1; + if (x1 > text1_length) { + // Ran off the right of the graph. + k1end += 2; + } else if (y1 > text2_length) { + // Ran off the bottom of the graph. + k1start += 2; + } else if (front) { + int k2_offset = v_offset + delta - k1; + if (k2_offset >= 0 && k2_offset < v_length && v2[k2_offset] != -1) { + // Mirror x2 onto top-left coordinate system. + int x2 = text1_length - v2[k2_offset]; + if (x1 >= x2) { + // Overlap detected. + return diff_bisectSplit(text1, text2, x1, y1, deadline); + } + } + } + } + + // Walk the reverse path one step. + for (int k2 = -d + k2start; k2 <= d - k2end; k2 += 2) { + int k2_offset = v_offset + k2; + int x2; + if (k2 == -d || (k2 != d && v2[k2_offset - 1] < v2[k2_offset + 1])) { + x2 = v2[k2_offset + 1]; + } else { + x2 = v2[k2_offset - 1] + 1; + } + int y2 = x2 - k2; + while (x2 < text1_length && y2 < text2_length + && text1.charAt(text1_length - x2 - 1) + == text2.charAt(text2_length - y2 - 1)) { + x2++; + y2++; + } + v2[k2_offset] = x2; + if (x2 > text1_length) { + // Ran off the left of the graph. + k2end += 2; + } else if (y2 > text2_length) { + // Ran off the top of the graph. + k2start += 2; + } else if (!front) { + int k1_offset = v_offset + delta - k2; + if (k1_offset >= 0 && k1_offset < v_length && v1[k1_offset] != -1) { + int x1 = v1[k1_offset]; + int y1 = v_offset + x1 - k1_offset; + // Mirror x2 onto top-left coordinate system. + x2 = text1_length - x2; + if (x1 >= x2) { + // Overlap detected. + return diff_bisectSplit(text1, text2, x1, y1, deadline); + } + } + } + } + } + // Diff took too long and hit the deadline or + // number of diffs equals number of characters, no commonality at all. + LinkedList diffs = new LinkedList(); + diffs.add(new Diff(Operation.DELETE, text1)); + diffs.add(new Diff(Operation.INSERT, text2)); + return diffs; + } + + /** + * Given the location of the 'middle snake', split the diff in two parts + * and recurse. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param x Index of split point in text1. + * @param y Index of split point in text2. + * @param deadline Time at which to bail if not yet complete. + * @return LinkedList of Diff objects. + */ + private LinkedList diff_bisectSplit(String text1, String text2, + int x, int y, long deadline) { + String text1a = text1.substring(0, x); + String text2a = text2.substring(0, y); + String text1b = text1.substring(x); + String text2b = text2.substring(y); + + // Compute both diffs serially. + LinkedList diffs = diff_main(text1a, text2a, false, deadline); + LinkedList diffsb = diff_main(text1b, text2b, false, deadline); + + diffs.addAll(diffsb); + return diffs; + } + + /** + * Split two texts into a list of strings. Reduce the texts to a string of + * hashes where each Unicode character represents one line. + * @param text1 First string. + * @param text2 Second string. + * @return An object containing the encoded text1, the encoded text2 and + * the List of unique strings. The zeroth element of the List of + * unique strings is intentionally blank. + */ + protected LinesToCharsResult diff_linesToChars(String text1, String text2) { + List lineArray = new ArrayList(); + Map lineHash = new HashMap(); + // e.g. linearray[4] == "Hello\n" + // e.g. linehash.get("Hello\n") == 4 + + // "\x00" is a valid character, but various debuggers don't like it. + // So we'll insert a junk entry to avoid generating a null character. + lineArray.add(""); + + // Allocate 2/3rds of the space for text1, the rest for text2. + String chars1 = diff_linesToCharsMunge(text1, lineArray, lineHash, 40000); + String chars2 = diff_linesToCharsMunge(text2, lineArray, lineHash, 65535); + return new LinesToCharsResult(chars1, chars2, lineArray); + } + + /** + * Split a text into a list of strings. Reduce the texts to a string of + * hashes where each Unicode character represents one line. + * @param text String to encode. + * @param lineArray List of unique strings. + * @param lineHash Map of strings to indices. + * @param maxLines Maximum length of lineArray. + * @return Encoded string. + */ + private String diff_linesToCharsMunge(String text, List lineArray, + Map lineHash, int maxLines) { + int lineStart = 0; + int lineEnd = -1; + String line; + StringBuilder chars = new StringBuilder(); + // Walk the text, pulling out a substring for each line. + // text.split('\n') would would temporarily double our memory footprint. + // Modifying text would create many large strings to garbage collect. + while (lineEnd < text.length() - 1) { + lineEnd = text.indexOf('\n', lineStart); + if (lineEnd == -1) { + lineEnd = text.length() - 1; + } + line = text.substring(lineStart, lineEnd + 1); + + if (lineHash.containsKey(line)) { + chars.append(String.valueOf((char) (int) lineHash.get(line))); + } else { + if (lineArray.size() == maxLines) { + // Bail out at 65535 because + // String.valueOf((char) 65536).equals(String.valueOf(((char) 0))) + line = text.substring(lineStart); + lineEnd = text.length(); + } + lineArray.add(line); + lineHash.put(line, lineArray.size() - 1); + chars.append(String.valueOf((char) (lineArray.size() - 1))); + } + lineStart = lineEnd + 1; + } + return chars.toString(); + } + + /** + * Rehydrate the text in a diff from a string of line hashes to real lines of + * text. + * @param diffs List of Diff objects. + * @param lineArray List of unique strings. + */ + protected void diff_charsToLines(List diffs, + List lineArray) { + StringBuilder text; + for (Diff diff : diffs) { + text = new StringBuilder(); + for (int j = 0; j < diff.text.length(); j++) { + text.append(lineArray.get(diff.text.charAt(j))); + } + diff.text = text.toString(); + } + } + + /** + * Determine the common prefix of two strings + * @param text1 First string. + * @param text2 Second string. + * @return The number of characters common to the start of each string. + */ + public int diff_commonPrefix(String text1, String text2) { + // Performance analysis: https://neil.fraser.name/news/2007/10/09/ + int n = Math.min(text1.length(), text2.length()); + for (int i = 0; i < n; i++) { + if (text1.charAt(i) != text2.charAt(i)) { + return i; + } + } + return n; + } + + /** + * Determine the common suffix of two strings + * @param text1 First string. + * @param text2 Second string. + * @return The number of characters common to the end of each string. + */ + public int diff_commonSuffix(String text1, String text2) { + // Performance analysis: https://neil.fraser.name/news/2007/10/09/ + int text1_length = text1.length(); + int text2_length = text2.length(); + int n = Math.min(text1_length, text2_length); + for (int i = 1; i <= n; i++) { + if (text1.charAt(text1_length - i) != text2.charAt(text2_length - i)) { + return i - 1; + } + } + return n; + } + + /** + * Determine if the suffix of one string is the prefix of another. + * @param text1 First string. + * @param text2 Second string. + * @return The number of characters common to the end of the first + * string and the start of the second string. + */ + protected int diff_commonOverlap(String text1, String text2) { + // Cache the text lengths to prevent multiple calls. + int text1_length = text1.length(); + int text2_length = text2.length(); + // Eliminate the null case. + if (text1_length == 0 || text2_length == 0) { + return 0; + } + // Truncate the longer string. + if (text1_length > text2_length) { + text1 = text1.substring(text1_length - text2_length); + } else if (text1_length < text2_length) { + text2 = text2.substring(0, text1_length); + } + int text_length = Math.min(text1_length, text2_length); + // Quick check for the worst case. + if (text1.equals(text2)) { + return text_length; + } + + // Start by looking for a single character match + // and increase length until no match is found. + // Performance analysis: https://neil.fraser.name/news/2010/11/04/ + int best = 0; + int length = 1; + while (true) { + String pattern = text1.substring(text_length - length); + int found = text2.indexOf(pattern); + if (found == -1) { + return best; + } + length += found; + if (found == 0 || text1.substring(text_length - length).equals( + text2.substring(0, length))) { + best = length; + length++; + } + } + } + + /** + * Do the two texts share a substring which is at least half the length of + * the longer text? + * This speedup can produce non-minimal diffs. + * @param text1 First string. + * @param text2 Second string. + * @return Five element String array, containing the prefix of text1, the + * suffix of text1, the prefix of text2, the suffix of text2 and the + * common middle. Or null if there was no match. + */ + protected String[] diff_halfMatch(String text1, String text2) { + if (Diff_Timeout <= 0) { + // Don't risk returning a non-optimal diff if we have unlimited time. + return null; + } + String longtext = text1.length() > text2.length() ? text1 : text2; + String shorttext = text1.length() > text2.length() ? text2 : text1; + if (longtext.length() < 4 || shorttext.length() * 2 < longtext.length()) { + return null; // Pointless. + } + + // First check if the second quarter is the seed for a half-match. + String[] hm1 = diff_halfMatchI(longtext, shorttext, + (longtext.length() + 3) / 4); + // Check again based on the third quarter. + String[] hm2 = diff_halfMatchI(longtext, shorttext, + (longtext.length() + 1) / 2); + String[] hm; + if (hm1 == null && hm2 == null) { + return null; + } else if (hm2 == null) { + hm = hm1; + } else if (hm1 == null) { + hm = hm2; + } else { + // Both matched. Select the longest. + hm = hm1[4].length() > hm2[4].length() ? hm1 : hm2; + } + + // A half-match was found, sort out the return data. + if (text1.length() > text2.length()) { + return hm; + //return new String[]{hm[0], hm[1], hm[2], hm[3], hm[4]}; + } else { + return new String[]{hm[2], hm[3], hm[0], hm[1], hm[4]}; + } + } + + /** + * Does a substring of shorttext exist within longtext such that the + * substring is at least half the length of longtext? + * @param longtext Longer string. + * @param shorttext Shorter string. + * @param i Start index of quarter length substring within longtext. + * @return Five element String array, containing the prefix of longtext, the + * suffix of longtext, the prefix of shorttext, the suffix of shorttext + * and the common middle. Or null if there was no match. + */ + private String[] diff_halfMatchI(String longtext, String shorttext, int i) { + // Start with a 1/4 length substring at position i as a seed. + String seed = longtext.substring(i, i + longtext.length() / 4); + int j = -1; + String best_common = ""; + String best_longtext_a = "", best_longtext_b = ""; + String best_shorttext_a = "", best_shorttext_b = ""; + while ((j = shorttext.indexOf(seed, j + 1)) != -1) { + int prefixLength = diff_commonPrefix(longtext.substring(i), + shorttext.substring(j)); + int suffixLength = diff_commonSuffix(longtext.substring(0, i), + shorttext.substring(0, j)); + if (best_common.length() < suffixLength + prefixLength) { + best_common = shorttext.substring(j - suffixLength, j) + + shorttext.substring(j, j + prefixLength); + best_longtext_a = longtext.substring(0, i - suffixLength); + best_longtext_b = longtext.substring(i + prefixLength); + best_shorttext_a = shorttext.substring(0, j - suffixLength); + best_shorttext_b = shorttext.substring(j + prefixLength); + } + } + if (best_common.length() * 2 >= longtext.length()) { + return new String[]{best_longtext_a, best_longtext_b, + best_shorttext_a, best_shorttext_b, best_common}; + } else { + return null; + } + } + + /** + * Reduce the number of edits by eliminating semantically trivial equalities. + * @param diffs LinkedList of Diff objects. + */ + public void diff_cleanupSemantic(LinkedList diffs) { + if (diffs.isEmpty()) { + return; + } + boolean changes = false; + Deque equalities = new ArrayDeque(); // Double-ended queue of qualities. + String lastEquality = null; // Always equal to equalities.peek().text + ListIterator pointer = diffs.listIterator(); + // Number of characters that changed prior to the equality. + int length_insertions1 = 0; + int length_deletions1 = 0; + // Number of characters that changed after the equality. + int length_insertions2 = 0; + int length_deletions2 = 0; + Diff thisDiff = pointer.next(); + while (thisDiff != null) { + if (thisDiff.operation == Operation.EQUAL) { + // Equality found. + equalities.push(thisDiff); + length_insertions1 = length_insertions2; + length_deletions1 = length_deletions2; + length_insertions2 = 0; + length_deletions2 = 0; + lastEquality = thisDiff.text; + } else { + // An insertion or deletion. + if (thisDiff.operation == Operation.INSERT) { + length_insertions2 += thisDiff.text.length(); + } else { + length_deletions2 += thisDiff.text.length(); + } + // Eliminate an equality that is smaller or equal to the edits on both + // sides of it. + if (lastEquality != null && (lastEquality.length() + <= Math.max(length_insertions1, length_deletions1)) + && (lastEquality.length() + <= Math.max(length_insertions2, length_deletions2))) { + //System.out.println("Splitting: '" + lastEquality + "'"); + // Walk back to offending equality. + while (thisDiff != equalities.peek()) { + thisDiff = pointer.previous(); + } + pointer.next(); + + // Replace equality with a delete. + pointer.set(new Diff(Operation.DELETE, lastEquality)); + // Insert a corresponding an insert. + pointer.add(new Diff(Operation.INSERT, lastEquality)); + + equalities.pop(); // Throw away the equality we just deleted. + if (!equalities.isEmpty()) { + // Throw away the previous equality (it needs to be reevaluated). + equalities.pop(); + } + if (equalities.isEmpty()) { + // There are no previous equalities, walk back to the start. + while (pointer.hasPrevious()) { + pointer.previous(); + } + } else { + // There is a safe equality we can fall back to. + thisDiff = equalities.peek(); + while (thisDiff != pointer.previous()) { + // Intentionally empty loop. + } + } + + length_insertions1 = 0; // Reset the counters. + length_insertions2 = 0; + length_deletions1 = 0; + length_deletions2 = 0; + lastEquality = null; + changes = true; + } + } + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + + // Normalize the diff. + if (changes) { + diff_cleanupMerge(diffs); + } + diff_cleanupSemanticLossless(diffs); + + // Find any overlaps between deletions and insertions. + // e.g: abcxxxxxxdef + // -> abcxxxdef + // e.g: xxxabcdefxxx + // -> defxxxabc + // Only extract an overlap if it is as big as the edit ahead or behind it. + pointer = diffs.listIterator(); + Diff prevDiff = null; + thisDiff = null; + if (pointer.hasNext()) { + prevDiff = pointer.next(); + if (pointer.hasNext()) { + thisDiff = pointer.next(); + } + } + while (thisDiff != null) { + if (prevDiff.operation == Operation.DELETE && + thisDiff.operation == Operation.INSERT) { + String deletion = prevDiff.text; + String insertion = thisDiff.text; + int overlap_length1 = this.diff_commonOverlap(deletion, insertion); + int overlap_length2 = this.diff_commonOverlap(insertion, deletion); + if (overlap_length1 >= overlap_length2) { + if (overlap_length1 >= deletion.length() / 2.0 || + overlap_length1 >= insertion.length() / 2.0) { + // Overlap found. Insert an equality and trim the surrounding edits. + pointer.previous(); + pointer.add(new Diff(Operation.EQUAL, + insertion.substring(0, overlap_length1))); + prevDiff.text = + deletion.substring(0, deletion.length() - overlap_length1); + thisDiff.text = insertion.substring(overlap_length1); + // pointer.add inserts the element before the cursor, so there is + // no need to step past the new element. + } + } else { + if (overlap_length2 >= deletion.length() / 2.0 || + overlap_length2 >= insertion.length() / 2.0) { + // Reverse overlap found. + // Insert an equality and swap and trim the surrounding edits. + pointer.previous(); + pointer.add(new Diff(Operation.EQUAL, + deletion.substring(0, overlap_length2))); + prevDiff.operation = Operation.INSERT; + prevDiff.text = + insertion.substring(0, insertion.length() - overlap_length2); + thisDiff.operation = Operation.DELETE; + thisDiff.text = deletion.substring(overlap_length2); + // pointer.add inserts the element before the cursor, so there is + // no need to step past the new element. + } + } + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + prevDiff = thisDiff; + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + } + + /** + * Look for single edits surrounded on both sides by equalities + * which can be shifted sideways to align the edit to a word boundary. + * e.g: The cat came. -> The cat came. + * @param diffs LinkedList of Diff objects. + */ + public void diff_cleanupSemanticLossless(LinkedList diffs) { + String equality1, edit, equality2; + String commonString; + int commonOffset; + int score, bestScore; + String bestEquality1, bestEdit, bestEquality2; + // Create a new iterator at the start. + ListIterator pointer = diffs.listIterator(); + Diff prevDiff = pointer.hasNext() ? pointer.next() : null; + Diff thisDiff = pointer.hasNext() ? pointer.next() : null; + Diff nextDiff = pointer.hasNext() ? pointer.next() : null; + // Intentionally ignore the first and last element (don't need checking). + while (nextDiff != null) { + if (prevDiff.operation == Operation.EQUAL && + nextDiff.operation == Operation.EQUAL) { + // This is a single edit surrounded by equalities. + equality1 = prevDiff.text; + edit = thisDiff.text; + equality2 = nextDiff.text; + + // First, shift the edit as far left as possible. + commonOffset = diff_commonSuffix(equality1, edit); + if (commonOffset != 0) { + commonString = edit.substring(edit.length() - commonOffset); + equality1 = equality1.substring(0, equality1.length() - commonOffset); + edit = commonString + edit.substring(0, edit.length() - commonOffset); + equality2 = commonString + equality2; + } + + // Second, step character by character right, looking for the best fit. + bestEquality1 = equality1; + bestEdit = edit; + bestEquality2 = equality2; + bestScore = diff_cleanupSemanticScore(equality1, edit) + + diff_cleanupSemanticScore(edit, equality2); + while (edit.length() != 0 && equality2.length() != 0 + && edit.charAt(0) == equality2.charAt(0)) { + equality1 += edit.charAt(0); + edit = edit.substring(1) + equality2.charAt(0); + equality2 = equality2.substring(1); + score = diff_cleanupSemanticScore(equality1, edit) + + diff_cleanupSemanticScore(edit, equality2); + // The >= encourages trailing rather than leading whitespace on edits. + if (score >= bestScore) { + bestScore = score; + bestEquality1 = equality1; + bestEdit = edit; + bestEquality2 = equality2; + } + } + + if (!prevDiff.text.equals(bestEquality1)) { + // We have an improvement, save it back to the diff. + if (bestEquality1.length() != 0) { + prevDiff.text = bestEquality1; + } else { + pointer.previous(); // Walk past nextDiff. + pointer.previous(); // Walk past thisDiff. + pointer.previous(); // Walk past prevDiff. + pointer.remove(); // Delete prevDiff. + pointer.next(); // Walk past thisDiff. + pointer.next(); // Walk past nextDiff. + } + thisDiff.text = bestEdit; + if (bestEquality2.length() != 0) { + nextDiff.text = bestEquality2; + } else { + pointer.remove(); // Delete nextDiff. + nextDiff = thisDiff; + thisDiff = prevDiff; + } + } + } + prevDiff = thisDiff; + thisDiff = nextDiff; + nextDiff = pointer.hasNext() ? pointer.next() : null; + } + } + + /** + * Given two strings, compute a score representing whether the internal + * boundary falls on logical boundaries. + * Scores range from 6 (best) to 0 (worst). + * @param one First string. + * @param two Second string. + * @return The score. + */ + private int diff_cleanupSemanticScore(String one, String two) { + if (one.length() == 0 || two.length() == 0) { + // Edges are the best. + return 6; + } + + // Each port of this function behaves slightly differently due to + // subtle differences in each language's definition of things like + // 'whitespace'. Since this function's purpose is largely cosmetic, + // the choice has been made to use each language's native features + // rather than force total conformity. + char char1 = one.charAt(one.length() - 1); + char char2 = two.charAt(0); + boolean nonAlphaNumeric1 = !Character.isLetterOrDigit(char1); + boolean nonAlphaNumeric2 = !Character.isLetterOrDigit(char2); + boolean whitespace1 = nonAlphaNumeric1 && Character.isWhitespace(char1); + boolean whitespace2 = nonAlphaNumeric2 && Character.isWhitespace(char2); + boolean lineBreak1 = whitespace1 + && Character.getType(char1) == Character.CONTROL; + boolean lineBreak2 = whitespace2 + && Character.getType(char2) == Character.CONTROL; + boolean blankLine1 = lineBreak1 && BLANKLINEEND.matcher(one).find(); + boolean blankLine2 = lineBreak2 && BLANKLINESTART.matcher(two).find(); + + if (blankLine1 || blankLine2) { + // Five points for blank lines. + return 5; + } else if (lineBreak1 || lineBreak2) { + // Four points for line breaks. + return 4; + } else if (nonAlphaNumeric1 && !whitespace1 && whitespace2) { + // Three points for end of sentences. + return 3; + } else if (whitespace1 || whitespace2) { + // Two points for whitespace. + return 2; + } else if (nonAlphaNumeric1 || nonAlphaNumeric2) { + // One point for non-alphanumeric. + return 1; + } + return 0; + } + + // Define some regex patterns for matching boundaries. + private Pattern BLANKLINEEND + = Pattern.compile("\\n\\r?\\n\\Z", Pattern.DOTALL); + private Pattern BLANKLINESTART + = Pattern.compile("\\A\\r?\\n\\r?\\n", Pattern.DOTALL); + + /** + * Reduce the number of edits by eliminating operationally trivial equalities. + * @param diffs LinkedList of Diff objects. + */ + public void diff_cleanupEfficiency(LinkedList diffs) { + if (diffs.isEmpty()) { + return; + } + boolean changes = false; + Deque equalities = new ArrayDeque(); // Double-ended queue of equalities. + String lastEquality = null; // Always equal to equalities.peek().text + ListIterator pointer = diffs.listIterator(); + // Is there an insertion operation before the last equality. + boolean pre_ins = false; + // Is there a deletion operation before the last equality. + boolean pre_del = false; + // Is there an insertion operation after the last equality. + boolean post_ins = false; + // Is there a deletion operation after the last equality. + boolean post_del = false; + Diff thisDiff = pointer.next(); + Diff safeDiff = thisDiff; // The last Diff that is known to be unsplittable. + while (thisDiff != null) { + if (thisDiff.operation == Operation.EQUAL) { + // Equality found. + if (thisDiff.text.length() < Diff_EditCost && (post_ins || post_del)) { + // Candidate found. + equalities.push(thisDiff); + pre_ins = post_ins; + pre_del = post_del; + lastEquality = thisDiff.text; + } else { + // Not a candidate, and can never become one. + equalities.clear(); + lastEquality = null; + safeDiff = thisDiff; + } + post_ins = post_del = false; + } else { + // An insertion or deletion. + if (thisDiff.operation == Operation.DELETE) { + post_del = true; + } else { + post_ins = true; + } + /* + * Five types to be split: + * ABXYCD + * AXCD + * ABXC + * AXCD + * ABXC + */ + if (lastEquality != null + && ((pre_ins && pre_del && post_ins && post_del) + || ((lastEquality.length() < Diff_EditCost / 2) + && ((pre_ins ? 1 : 0) + (pre_del ? 1 : 0) + + (post_ins ? 1 : 0) + (post_del ? 1 : 0)) == 3))) { + //System.out.println("Splitting: '" + lastEquality + "'"); + // Walk back to offending equality. + while (thisDiff != equalities.peek()) { + thisDiff = pointer.previous(); + } + pointer.next(); + + // Replace equality with a delete. + pointer.set(new Diff(Operation.DELETE, lastEquality)); + // Insert a corresponding an insert. + pointer.add(thisDiff = new Diff(Operation.INSERT, lastEquality)); + + equalities.pop(); // Throw away the equality we just deleted. + lastEquality = null; + if (pre_ins && pre_del) { + // No changes made which could affect previous entry, keep going. + post_ins = post_del = true; + equalities.clear(); + safeDiff = thisDiff; + } else { + if (!equalities.isEmpty()) { + // Throw away the previous equality (it needs to be reevaluated). + equalities.pop(); + } + if (equalities.isEmpty()) { + // There are no previous questionable equalities, + // walk back to the last known safe diff. + thisDiff = safeDiff; + } else { + // There is an equality we can fall back to. + thisDiff = equalities.peek(); + } + while (thisDiff != pointer.previous()) { + // Intentionally empty loop. + } + post_ins = post_del = false; + } + + changes = true; + } + } + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + + if (changes) { + diff_cleanupMerge(diffs); + } + } + + /** + * Reorder and merge like edit sections. Merge equalities. + * Any edit section can move as long as it doesn't cross an equality. + * @param diffs LinkedList of Diff objects. + */ + public void diff_cleanupMerge(LinkedList diffs) { + diffs.add(new Diff(Operation.EQUAL, "")); // Add a dummy entry at the end. + ListIterator pointer = diffs.listIterator(); + int count_delete = 0; + int count_insert = 0; + String text_delete = ""; + String text_insert = ""; + Diff thisDiff = pointer.next(); + Diff prevEqual = null; + int commonlength; + while (thisDiff != null) { + switch (thisDiff.operation) { + case INSERT: + count_insert++; + text_insert += thisDiff.text; + prevEqual = null; + break; + case DELETE: + count_delete++; + text_delete += thisDiff.text; + prevEqual = null; + break; + case EQUAL: + if (count_delete + count_insert > 1) { + boolean both_types = count_delete != 0 && count_insert != 0; + // Delete the offending records. + pointer.previous(); // Reverse direction. + while (count_delete-- > 0) { + pointer.previous(); + pointer.remove(); + } + while (count_insert-- > 0) { + pointer.previous(); + pointer.remove(); + } + if (both_types) { + // Factor out any common prefixies. + commonlength = diff_commonPrefix(text_insert, text_delete); + if (commonlength != 0) { + if (pointer.hasPrevious()) { + thisDiff = pointer.previous(); + assert thisDiff.operation == Operation.EQUAL + : "Previous diff should have been an equality."; + thisDiff.text += text_insert.substring(0, commonlength); + pointer.next(); + } else { + pointer.add(new Diff(Operation.EQUAL, + text_insert.substring(0, commonlength))); + } + text_insert = text_insert.substring(commonlength); + text_delete = text_delete.substring(commonlength); + } + // Factor out any common suffixies. + commonlength = diff_commonSuffix(text_insert, text_delete); + if (commonlength != 0) { + thisDiff = pointer.next(); + thisDiff.text = text_insert.substring(text_insert.length() + - commonlength) + thisDiff.text; + text_insert = text_insert.substring(0, text_insert.length() + - commonlength); + text_delete = text_delete.substring(0, text_delete.length() + - commonlength); + pointer.previous(); + } + } + // Insert the merged records. + if (text_delete.length() != 0) { + pointer.add(new Diff(Operation.DELETE, text_delete)); + } + if (text_insert.length() != 0) { + pointer.add(new Diff(Operation.INSERT, text_insert)); + } + // Step forward to the equality. + thisDiff = pointer.hasNext() ? pointer.next() : null; + } else if (prevEqual != null) { + // Merge this equality with the previous one. + prevEqual.text += thisDiff.text; + pointer.remove(); + thisDiff = pointer.previous(); + pointer.next(); // Forward direction + } + count_insert = 0; + count_delete = 0; + text_delete = ""; + text_insert = ""; + prevEqual = thisDiff; + break; + } + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + if (diffs.getLast().text.length() == 0) { + diffs.removeLast(); // Remove the dummy entry at the end. + } + + /* + * Second pass: look for single edits surrounded on both sides by equalities + * which can be shifted sideways to eliminate an equality. + * e.g: ABAC -> ABAC + */ + boolean changes = false; + // Create a new iterator at the start. + // (As opposed to walking the current one back.) + pointer = diffs.listIterator(); + Diff prevDiff = pointer.hasNext() ? pointer.next() : null; + thisDiff = pointer.hasNext() ? pointer.next() : null; + Diff nextDiff = pointer.hasNext() ? pointer.next() : null; + // Intentionally ignore the first and last element (don't need checking). + while (nextDiff != null) { + if (prevDiff.operation == Operation.EQUAL && + nextDiff.operation == Operation.EQUAL) { + // This is a single edit surrounded by equalities. + if (thisDiff.text.endsWith(prevDiff.text)) { + // Shift the edit over the previous equality. + thisDiff.text = prevDiff.text + + thisDiff.text.substring(0, thisDiff.text.length() + - prevDiff.text.length()); + nextDiff.text = prevDiff.text + nextDiff.text; + pointer.previous(); // Walk past nextDiff. + pointer.previous(); // Walk past thisDiff. + pointer.previous(); // Walk past prevDiff. + pointer.remove(); // Delete prevDiff. + pointer.next(); // Walk past thisDiff. + thisDiff = pointer.next(); // Walk past nextDiff. + nextDiff = pointer.hasNext() ? pointer.next() : null; + changes = true; + } else if (thisDiff.text.startsWith(nextDiff.text)) { + // Shift the edit over the next equality. + prevDiff.text += nextDiff.text; + thisDiff.text = thisDiff.text.substring(nextDiff.text.length()) + + nextDiff.text; + pointer.remove(); // Delete nextDiff. + nextDiff = pointer.hasNext() ? pointer.next() : null; + changes = true; + } + } + prevDiff = thisDiff; + thisDiff = nextDiff; + nextDiff = pointer.hasNext() ? pointer.next() : null; + } + // If shifts were made, the diff needs reordering and another shift sweep. + if (changes) { + diff_cleanupMerge(diffs); + } + } + + /** + * loc is a location in text1, compute and return the equivalent location in + * text2. + * e.g. "The cat" vs "The big cat", 1->1, 5->8 + * @param diffs List of Diff objects. + * @param loc Location within text1. + * @return Location within text2. + */ + public int diff_xIndex(List diffs, int loc) { + int chars1 = 0; + int chars2 = 0; + int last_chars1 = 0; + int last_chars2 = 0; + Diff lastDiff = null; + for (Diff aDiff : diffs) { + if (aDiff.operation != Operation.INSERT) { + // Equality or deletion. + chars1 += aDiff.text.length(); + } + if (aDiff.operation != Operation.DELETE) { + // Equality or insertion. + chars2 += aDiff.text.length(); + } + if (chars1 > loc) { + // Overshot the location. + lastDiff = aDiff; + break; + } + last_chars1 = chars1; + last_chars2 = chars2; + } + if (lastDiff != null && lastDiff.operation == Operation.DELETE) { + // The location was deleted. + return last_chars2; + } + // Add the remaining character length. + return last_chars2 + (loc - last_chars1); + } + + /** + * Convert a Diff list into a pretty HTML report. + * @param diffs List of Diff objects. + * @return HTML representation. + */ + public String diff_prettyHtml(List diffs) { + StringBuilder html = new StringBuilder(); + for (Diff aDiff : diffs) { + String text = aDiff.text.replace("&", "&").replace("<", "<") + .replace(">", ">").replace("\n", "¶
"); + switch (aDiff.operation) { + case INSERT: + html.append("").append(text) + .append(""); + break; + case DELETE: + html.append("").append(text) + .append(""); + break; + case EQUAL: + html.append("").append(text).append(""); + break; + } + } + return html.toString(); + } + + /** + * Compute and return the source text (all equalities and deletions). + * @param diffs List of Diff objects. + * @return Source text. + */ + public String diff_text1(List diffs) { + StringBuilder text = new StringBuilder(); + for (Diff aDiff : diffs) { + if (aDiff.operation != Operation.INSERT) { + text.append(aDiff.text); + } + } + return text.toString(); + } + + /** + * Compute and return the destination text (all equalities and insertions). + * @param diffs List of Diff objects. + * @return Destination text. + */ + public String diff_text2(List diffs) { + StringBuilder text = new StringBuilder(); + for (Diff aDiff : diffs) { + if (aDiff.operation != Operation.DELETE) { + text.append(aDiff.text); + } + } + return text.toString(); + } + + /** + * Compute the Levenshtein distance; the number of inserted, deleted or + * substituted characters. + * @param diffs List of Diff objects. + * @return Number of changes. + */ + public int diff_levenshtein(List diffs) { + int levenshtein = 0; + int insertions = 0; + int deletions = 0; + for (Diff aDiff : diffs) { + switch (aDiff.operation) { + case INSERT: + insertions += aDiff.text.length(); + break; + case DELETE: + deletions += aDiff.text.length(); + break; + case EQUAL: + // A deletion and an insertion is one substitution. + levenshtein += Math.max(insertions, deletions); + insertions = 0; + deletions = 0; + break; + } + } + levenshtein += Math.max(insertions, deletions); + return levenshtein; + } + + /** + * Crush the diff into an encoded string which describes the operations + * required to transform text1 into text2. + * E.g. =3\t-2\t+ing -> Keep 3 chars, delete 2 chars, insert 'ing'. + * Operations are tab-separated. Inserted text is escaped using %xx notation. + * @param diffs List of Diff objects. + * @return Delta text. + */ + public String diff_toDelta(List diffs) { + StringBuilder text = new StringBuilder(); + for (Diff aDiff : diffs) { + switch (aDiff.operation) { + case INSERT: + try { + text.append("+").append(URLEncoder.encode(aDiff.text, "UTF-8") + .replace('+', ' ')).append("\t"); + } catch (UnsupportedEncodingException e) { + // Not likely on modern system. + throw new Error("This system does not support UTF-8.", e); + } + break; + case DELETE: + text.append("-").append(aDiff.text.length()).append("\t"); + break; + case EQUAL: + text.append("=").append(aDiff.text.length()).append("\t"); + break; + } + } + String delta = text.toString(); + if (delta.length() != 0) { + // Strip off trailing tab character. + delta = delta.substring(0, delta.length() - 1); + delta = unescapeForEncodeUriCompatability(delta); + } + return delta; + } + + /** + * Given the original text1, and an encoded string which describes the + * operations required to transform text1 into text2, compute the full diff. + * @param text1 Source string for the diff. + * @param delta Delta text. + * @return Array of Diff objects or null if invalid. + * @throws IllegalArgumentException If invalid input. + */ + public LinkedList diff_fromDelta(String text1, String delta) + throws IllegalArgumentException { + LinkedList diffs = new LinkedList(); + int pointer = 0; // Cursor in text1 + String[] tokens = delta.split("\t"); + for (String token : tokens) { + if (token.length() == 0) { + // Blank tokens are ok (from a trailing \t). + continue; + } + // Each token begins with a one character parameter which specifies the + // operation of this token (delete, insert, equality). + String param = token.substring(1); + switch (token.charAt(0)) { + case '+': + // decode would change all "+" to " " + param = param.replace("+", "%2B"); + try { + param = URLDecoder.decode(param, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // Not likely on modern system. + throw new Error("This system does not support UTF-8.", e); + } catch (IllegalArgumentException e) { + // Malformed URI sequence. + throw new IllegalArgumentException( + "Illegal escape in diff_fromDelta: " + param, e); + } + diffs.add(new Diff(Operation.INSERT, param)); + break; + case '-': + // Fall through. + case '=': + int n; + try { + n = Integer.parseInt(param); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Invalid number in diff_fromDelta: " + param, e); + } + if (n < 0) { + throw new IllegalArgumentException( + "Negative number in diff_fromDelta: " + param); + } + String text; + try { + text = text1.substring(pointer, pointer += n); + } catch (StringIndexOutOfBoundsException e) { + throw new IllegalArgumentException("Delta length (" + pointer + + ") larger than source text length (" + text1.length() + + ").", e); + } + if (token.charAt(0) == '=') { + diffs.add(new Diff(Operation.EQUAL, text)); + } else { + diffs.add(new Diff(Operation.DELETE, text)); + } + break; + default: + // Anything else is an error. + throw new IllegalArgumentException( + "Invalid diff operation in diff_fromDelta: " + token.charAt(0)); + } + } + if (pointer != text1.length()) { + throw new IllegalArgumentException("Delta length (" + pointer + + ") smaller than source text length (" + text1.length() + ")."); + } + return diffs; + } + + + // MATCH FUNCTIONS + + + /** + * Locate the best instance of 'pattern' in 'text' near 'loc'. + * Returns -1 if no match found. + * @param text The text to search. + * @param pattern The pattern to search for. + * @param loc The location to search around. + * @return Best match index or -1. + */ + public int match_main(String text, String pattern, int loc) { + // Check for null inputs. + if (text == null || pattern == null) { + throw new IllegalArgumentException("Null inputs. (match_main)"); + } + + loc = Math.max(0, Math.min(loc, text.length())); + if (text.equals(pattern)) { + // Shortcut (potentially not guaranteed by the algorithm) + return 0; + } else if (text.length() == 0) { + // Nothing to match. + return -1; + } else if (loc + pattern.length() <= text.length() + && text.substring(loc, loc + pattern.length()).equals(pattern)) { + // Perfect match at the perfect spot! (Includes case of null pattern) + return loc; + } else { + // Do a fuzzy compare. + return match_bitap(text, pattern, loc); + } + } + + /** + * Locate the best instance of 'pattern' in 'text' near 'loc' using the + * Bitap algorithm. Returns -1 if no match found. + * @param text The text to search. + * @param pattern The pattern to search for. + * @param loc The location to search around. + * @return Best match index or -1. + */ + protected int match_bitap(String text, String pattern, int loc) { + assert (Match_MaxBits == 0 || pattern.length() <= Match_MaxBits) + : "Pattern too long for this application."; + + // Initialise the alphabet. + Map s = match_alphabet(pattern); + + // Highest score beyond which we give up. + double score_threshold = Match_Threshold; + // Is there a nearby exact match? (speedup) + int best_loc = text.indexOf(pattern, loc); + if (best_loc != -1) { + score_threshold = Math.min(match_bitapScore(0, best_loc, loc, pattern), + score_threshold); + // What about in the other direction? (speedup) + best_loc = text.lastIndexOf(pattern, loc + pattern.length()); + if (best_loc != -1) { + score_threshold = Math.min(match_bitapScore(0, best_loc, loc, pattern), + score_threshold); + } + } + + // Initialise the bit arrays. + int matchmask = 1 << (pattern.length() - 1); + best_loc = -1; + + int bin_min, bin_mid; + int bin_max = pattern.length() + text.length(); + // Empty initialization added to appease Java compiler. + int[] last_rd = new int[0]; + for (int d = 0; d < pattern.length(); d++) { + // Scan for the best match; each iteration allows for one more error. + // Run a binary search to determine how far from 'loc' we can stray at + // this error level. + bin_min = 0; + bin_mid = bin_max; + while (bin_min < bin_mid) { + if (match_bitapScore(d, loc + bin_mid, loc, pattern) + <= score_threshold) { + bin_min = bin_mid; + } else { + bin_max = bin_mid; + } + bin_mid = (bin_max - bin_min) / 2 + bin_min; + } + // Use the result from this iteration as the maximum for the next. + bin_max = bin_mid; + int start = Math.max(1, loc - bin_mid + 1); + int finish = Math.min(loc + bin_mid, text.length()) + pattern.length(); + + int[] rd = new int[finish + 2]; + rd[finish + 1] = (1 << d) - 1; + for (int j = finish; j >= start; j--) { + int charMatch; + if (text.length() <= j - 1 || !s.containsKey(text.charAt(j - 1))) { + // Out of range. + charMatch = 0; + } else { + charMatch = s.get(text.charAt(j - 1)); + } + if (d == 0) { + // First pass: exact match. + rd[j] = ((rd[j + 1] << 1) | 1) & charMatch; + } else { + // Subsequent passes: fuzzy match. + rd[j] = (((rd[j + 1] << 1) | 1) & charMatch) + | (((last_rd[j + 1] | last_rd[j]) << 1) | 1) | last_rd[j + 1]; + } + if ((rd[j] & matchmask) != 0) { + double score = match_bitapScore(d, j - 1, loc, pattern); + // This match will almost certainly be better than any existing + // match. But check anyway. + if (score <= score_threshold) { + // Told you so. + score_threshold = score; + best_loc = j - 1; + if (best_loc > loc) { + // When passing loc, don't exceed our current distance from loc. + start = Math.max(1, 2 * loc - best_loc); + } else { + // Already passed loc, downhill from here on in. + break; + } + } + } + } + if (match_bitapScore(d + 1, loc, loc, pattern) > score_threshold) { + // No hope for a (better) match at greater error levels. + break; + } + last_rd = rd; + } + return best_loc; + } + + /** + * Compute and return the score for a match with e errors and x location. + * @param e Number of errors in match. + * @param x Location of match. + * @param loc Expected location of match. + * @param pattern Pattern being sought. + * @return Overall score for match (0.0 = good, 1.0 = bad). + */ + private double match_bitapScore(int e, int x, int loc, String pattern) { + float accuracy = (float) e / pattern.length(); + int proximity = Math.abs(loc - x); + if (Match_Distance == 0) { + // Dodge divide by zero error. + return proximity == 0 ? accuracy : 1.0; + } + return accuracy + (proximity / (float) Match_Distance); + } + + /** + * Initialise the alphabet for the Bitap algorithm. + * @param pattern The text to encode. + * @return Hash of character locations. + */ + protected Map match_alphabet(String pattern) { + Map s = new HashMap(); + char[] char_pattern = pattern.toCharArray(); + for (char c : char_pattern) { + s.put(c, 0); + } + int i = 0; + for (char c : char_pattern) { + s.put(c, s.get(c) | (1 << (pattern.length() - i - 1))); + i++; + } + return s; + } + + + // PATCH FUNCTIONS + + + /** + * Increase the context until it is unique, + * but don't let the pattern expand beyond Match_MaxBits. + * @param patch The patch to grow. + * @param text Source text. + */ + protected void patch_addContext(Patch patch, String text) { + if (text.length() == 0) { + return; + } + String pattern = text.substring(patch.start2, patch.start2 + patch.length1); + int padding = 0; + + // Look for the first and last matches of pattern in text. If two different + // matches are found, increase the pattern length. + while (text.indexOf(pattern) != text.lastIndexOf(pattern) + && pattern.length() < Match_MaxBits - Patch_Margin - Patch_Margin) { + padding += Patch_Margin; + pattern = text.substring(Math.max(0, patch.start2 - padding), + Math.min(text.length(), patch.start2 + patch.length1 + padding)); + } + // Add one chunk for good luck. + padding += Patch_Margin; + + // Add the prefix. + String prefix = text.substring(Math.max(0, patch.start2 - padding), + patch.start2); + if (prefix.length() != 0) { + patch.diffs.addFirst(new Diff(Operation.EQUAL, prefix)); + } + // Add the suffix. + String suffix = text.substring(patch.start2 + patch.length1, + Math.min(text.length(), patch.start2 + patch.length1 + padding)); + if (suffix.length() != 0) { + patch.diffs.addLast(new Diff(Operation.EQUAL, suffix)); + } + + // Roll back the start points. + patch.start1 -= prefix.length(); + patch.start2 -= prefix.length(); + // Extend the lengths. + patch.length1 += prefix.length() + suffix.length(); + patch.length2 += prefix.length() + suffix.length(); + } + + /** + * Compute a list of patches to turn text1 into text2. + * A set of diffs will be computed. + * @param text1 Old text. + * @param text2 New text. + * @return LinkedList of Patch objects. + */ + public LinkedList patch_make(String text1, String text2) { + if (text1 == null || text2 == null) { + throw new IllegalArgumentException("Null inputs. (patch_make)"); + } + // No diffs provided, compute our own. + LinkedList diffs = diff_main(text1, text2, true); + if (diffs.size() > 2) { + diff_cleanupSemantic(diffs); + diff_cleanupEfficiency(diffs); + } + return patch_make(text1, diffs); + } + + /** + * Compute a list of patches to turn text1 into text2. + * text1 will be derived from the provided diffs. + * @param diffs Array of Diff objects for text1 to text2. + * @return LinkedList of Patch objects. + */ + public LinkedList patch_make(LinkedList diffs) { + if (diffs == null) { + throw new IllegalArgumentException("Null inputs. (patch_make)"); + } + // No origin string provided, compute our own. + String text1 = diff_text1(diffs); + return patch_make(text1, diffs); + } + + /** + * Compute a list of patches to turn text1 into text2. + * text2 is ignored, diffs are the delta between text1 and text2. + * @param text1 Old text + * @param text2 Ignored. + * @param diffs Array of Diff objects for text1 to text2. + * @return LinkedList of Patch objects. + * @deprecated Prefer patch_make(String text1, LinkedList diffs). + */ + @Deprecated public LinkedList patch_make(String text1, String text2, + LinkedList diffs) { + return patch_make(text1, diffs); + } + + /** + * Compute a list of patches to turn text1 into text2. + * text2 is not provided, diffs are the delta between text1 and text2. + * @param text1 Old text. + * @param diffs Array of Diff objects for text1 to text2. + * @return LinkedList of Patch objects. + */ + public LinkedList patch_make(String text1, LinkedList diffs) { + if (text1 == null || diffs == null) { + throw new IllegalArgumentException("Null inputs. (patch_make)"); + } + + LinkedList patches = new LinkedList(); + if (diffs.isEmpty()) { + return patches; // Get rid of the null case. + } + Patch patch = new Patch(); + int char_count1 = 0; // Number of characters into the text1 string. + int char_count2 = 0; // Number of characters into the text2 string. + // Start with text1 (prepatch_text) and apply the diffs until we arrive at + // text2 (postpatch_text). We recreate the patches one by one to determine + // context info. + String prepatch_text = text1; + String postpatch_text = text1; + for (Diff aDiff : diffs) { + if (patch.diffs.isEmpty() && aDiff.operation != Operation.EQUAL) { + // A new patch starts here. + patch.start1 = char_count1; + patch.start2 = char_count2; + } + + switch (aDiff.operation) { + case INSERT: + patch.diffs.add(aDiff); + patch.length2 += aDiff.text.length(); + postpatch_text = postpatch_text.substring(0, char_count2) + + aDiff.text + postpatch_text.substring(char_count2); + break; + case DELETE: + patch.length1 += aDiff.text.length(); + patch.diffs.add(aDiff); + postpatch_text = postpatch_text.substring(0, char_count2) + + postpatch_text.substring(char_count2 + aDiff.text.length()); + break; + case EQUAL: + if (aDiff.text.length() <= 2 * Patch_Margin + && !patch.diffs.isEmpty() && aDiff != diffs.getLast()) { + // Small equality inside a patch. + patch.diffs.add(aDiff); + patch.length1 += aDiff.text.length(); + patch.length2 += aDiff.text.length(); + } + + if (aDiff.text.length() >= 2 * Patch_Margin && !patch.diffs.isEmpty()) { + // Time for a new patch. + if (!patch.diffs.isEmpty()) { + patch_addContext(patch, prepatch_text); + patches.add(patch); + patch = new Patch(); + // Unlike Unidiff, our patch lists have a rolling context. + // https://github.com/google/diff-match-patch/wiki/Unidiff + // Update prepatch text & pos to reflect the application of the + // just completed patch. + prepatch_text = postpatch_text; + char_count1 = char_count2; + } + } + break; + } + + // Update the current character count. + if (aDiff.operation != Operation.INSERT) { + char_count1 += aDiff.text.length(); + } + if (aDiff.operation != Operation.DELETE) { + char_count2 += aDiff.text.length(); + } + } + // Pick up the leftover patch if not empty. + if (!patch.diffs.isEmpty()) { + patch_addContext(patch, prepatch_text); + patches.add(patch); + } + + return patches; + } + + /** + * Given an array of patches, return another array that is identical. + * @param patches Array of Patch objects. + * @return Array of Patch objects. + */ + public LinkedList patch_deepCopy(LinkedList patches) { + LinkedList patchesCopy = new LinkedList(); + for (Patch aPatch : patches) { + Patch patchCopy = new Patch(); + for (Diff aDiff : aPatch.diffs) { + Diff diffCopy = new Diff(aDiff.operation, aDiff.text); + patchCopy.diffs.add(diffCopy); + } + patchCopy.start1 = aPatch.start1; + patchCopy.start2 = aPatch.start2; + patchCopy.length1 = aPatch.length1; + patchCopy.length2 = aPatch.length2; + patchesCopy.add(patchCopy); + } + return patchesCopy; + } + + /** + * Merge a set of patches onto the text. Return a patched text, as well + * as an array of true/false values indicating which patches were applied. + * @param patches Array of Patch objects + * @param text Old text. + * @return Two element Object array, containing the new text and an array of + * boolean values. + */ + public Object[] patch_apply(LinkedList patches, String text) { + if (patches.isEmpty()) { + return new Object[]{text, new boolean[0]}; + } + + // Deep copy the patches so that no changes are made to originals. + patches = patch_deepCopy(patches); + + String nullPadding = patch_addPadding(patches); + text = nullPadding + text + nullPadding; + patch_splitMax(patches); + + int x = 0; + // delta keeps track of the offset between the expected and actual location + // of the previous patch. If there are patches expected at positions 10 and + // 20, but the first patch was found at 12, delta is 2 and the second patch + // has an effective expected position of 22. + int delta = 0; + boolean[] results = new boolean[patches.size()]; + for (Patch aPatch : patches) { + int expected_loc = aPatch.start2 + delta; + String text1 = diff_text1(aPatch.diffs); + int start_loc; + int end_loc = -1; + if (text1.length() > this.Match_MaxBits) { + // patch_splitMax will only provide an oversized pattern in the case of + // a monster delete. + start_loc = match_main(text, + text1.substring(0, this.Match_MaxBits), expected_loc); + if (start_loc != -1) { + end_loc = match_main(text, + text1.substring(text1.length() - this.Match_MaxBits), + expected_loc + text1.length() - this.Match_MaxBits); + if (end_loc == -1 || start_loc >= end_loc) { + // Can't find valid trailing context. Drop this patch. + start_loc = -1; + } + } + } else { + start_loc = match_main(text, text1, expected_loc); + } + if (start_loc == -1) { + // No match found. :( + results[x] = false; + // Subtract the delta for this failed patch from subsequent patches. + delta -= aPatch.length2 - aPatch.length1; + } else { + // Found a match. :) + results[x] = true; + delta = start_loc - expected_loc; + String text2; + if (end_loc == -1) { + text2 = text.substring(start_loc, + Math.min(start_loc + text1.length(), text.length())); + } else { + text2 = text.substring(start_loc, + Math.min(end_loc + this.Match_MaxBits, text.length())); + } + if (text1.equals(text2)) { + // Perfect match, just shove the replacement text in. + text = text.substring(0, start_loc) + diff_text2(aPatch.diffs) + + text.substring(start_loc + text1.length()); + } else { + // Imperfect match. Run a diff to get a framework of equivalent + // indices. + LinkedList diffs = diff_main(text1, text2, false); + if (text1.length() > this.Match_MaxBits + && diff_levenshtein(diffs) / (float) text1.length() + > this.Patch_DeleteThreshold) { + // The end points match, but the content is unacceptably bad. + results[x] = false; + } else { + diff_cleanupSemanticLossless(diffs); + int index1 = 0; + for (Diff aDiff : aPatch.diffs) { + if (aDiff.operation != Operation.EQUAL) { + int index2 = diff_xIndex(diffs, index1); + if (aDiff.operation == Operation.INSERT) { + // Insertion + text = text.substring(0, start_loc + index2) + aDiff.text + + text.substring(start_loc + index2); + } else if (aDiff.operation == Operation.DELETE) { + // Deletion + text = text.substring(0, start_loc + index2) + + text.substring(start_loc + diff_xIndex(diffs, + index1 + aDiff.text.length())); + } + } + if (aDiff.operation != Operation.DELETE) { + index1 += aDiff.text.length(); + } + } + } + } + } + x++; + } + // Strip the padding off. + text = text.substring(nullPadding.length(), text.length() + - nullPadding.length()); + return new Object[]{text, results}; + } + + /** + * Add some padding on text start and end so that edges can match something. + * Intended to be called only from within patch_apply. + * @param patches Array of Patch objects. + * @return The padding string added to each side. + */ + public String patch_addPadding(LinkedList patches) { + short paddingLength = this.Patch_Margin; + String nullPadding = ""; + for (short x = 1; x <= paddingLength; x++) { + nullPadding += String.valueOf((char) x); + } + + // Bump all the patches forward. + for (Patch aPatch : patches) { + aPatch.start1 += paddingLength; + aPatch.start2 += paddingLength; + } + + // Add some padding on start of first diff. + Patch patch = patches.getFirst(); + LinkedList diffs = patch.diffs; + if (diffs.isEmpty() || diffs.getFirst().operation != Operation.EQUAL) { + // Add nullPadding equality. + diffs.addFirst(new Diff(Operation.EQUAL, nullPadding)); + patch.start1 -= paddingLength; // Should be 0. + patch.start2 -= paddingLength; // Should be 0. + patch.length1 += paddingLength; + patch.length2 += paddingLength; + } else if (paddingLength > diffs.getFirst().text.length()) { + // Grow first equality. + Diff firstDiff = diffs.getFirst(); + int extraLength = paddingLength - firstDiff.text.length(); + firstDiff.text = nullPadding.substring(firstDiff.text.length()) + + firstDiff.text; + patch.start1 -= extraLength; + patch.start2 -= extraLength; + patch.length1 += extraLength; + patch.length2 += extraLength; + } + + // Add some padding on end of last diff. + patch = patches.getLast(); + diffs = patch.diffs; + if (diffs.isEmpty() || diffs.getLast().operation != Operation.EQUAL) { + // Add nullPadding equality. + diffs.addLast(new Diff(Operation.EQUAL, nullPadding)); + patch.length1 += paddingLength; + patch.length2 += paddingLength; + } else if (paddingLength > diffs.getLast().text.length()) { + // Grow last equality. + Diff lastDiff = diffs.getLast(); + int extraLength = paddingLength - lastDiff.text.length(); + lastDiff.text += nullPadding.substring(0, extraLength); + patch.length1 += extraLength; + patch.length2 += extraLength; + } + + return nullPadding; + } + + /** + * Look through the patches and break up any which are longer than the + * maximum limit of the match algorithm. + * Intended to be called only from within patch_apply. + * @param patches LinkedList of Patch objects. + */ + public void patch_splitMax(LinkedList patches) { + short patch_size = Match_MaxBits; + String precontext, postcontext; + Patch patch; + int start1, start2; + boolean empty; + Operation diff_type; + String diff_text; + ListIterator pointer = patches.listIterator(); + Patch bigpatch = pointer.hasNext() ? pointer.next() : null; + while (bigpatch != null) { + if (bigpatch.length1 <= Match_MaxBits) { + bigpatch = pointer.hasNext() ? pointer.next() : null; + continue; + } + // Remove the big old patch. + pointer.remove(); + start1 = bigpatch.start1; + start2 = bigpatch.start2; + precontext = ""; + while (!bigpatch.diffs.isEmpty()) { + // Create one of several smaller patches. + patch = new Patch(); + empty = true; + patch.start1 = start1 - precontext.length(); + patch.start2 = start2 - precontext.length(); + if (precontext.length() != 0) { + patch.length1 = patch.length2 = precontext.length(); + patch.diffs.add(new Diff(Operation.EQUAL, precontext)); + } + while (!bigpatch.diffs.isEmpty() + && patch.length1 < patch_size - Patch_Margin) { + diff_type = bigpatch.diffs.getFirst().operation; + diff_text = bigpatch.diffs.getFirst().text; + if (diff_type == Operation.INSERT) { + // Insertions are harmless. + patch.length2 += diff_text.length(); + start2 += diff_text.length(); + patch.diffs.addLast(bigpatch.diffs.removeFirst()); + empty = false; + } else if (diff_type == Operation.DELETE && patch.diffs.size() == 1 + && patch.diffs.getFirst().operation == Operation.EQUAL + && diff_text.length() > 2 * patch_size) { + // This is a large deletion. Let it pass in one chunk. + patch.length1 += diff_text.length(); + start1 += diff_text.length(); + empty = false; + patch.diffs.add(new Diff(diff_type, diff_text)); + bigpatch.diffs.removeFirst(); + } else { + // Deletion or equality. Only take as much as we can stomach. + diff_text = diff_text.substring(0, Math.min(diff_text.length(), + patch_size - patch.length1 - Patch_Margin)); + patch.length1 += diff_text.length(); + start1 += diff_text.length(); + if (diff_type == Operation.EQUAL) { + patch.length2 += diff_text.length(); + start2 += diff_text.length(); + } else { + empty = false; + } + patch.diffs.add(new Diff(diff_type, diff_text)); + if (diff_text.equals(bigpatch.diffs.getFirst().text)) { + bigpatch.diffs.removeFirst(); + } else { + bigpatch.diffs.getFirst().text = bigpatch.diffs.getFirst().text + .substring(diff_text.length()); + } + } + } + // Compute the head context for the next patch. + precontext = diff_text2(patch.diffs); + precontext = precontext.substring(Math.max(0, precontext.length() + - Patch_Margin)); + // Append the end context for this patch. + if (diff_text1(bigpatch.diffs).length() > Patch_Margin) { + postcontext = diff_text1(bigpatch.diffs).substring(0, Patch_Margin); + } else { + postcontext = diff_text1(bigpatch.diffs); + } + if (postcontext.length() != 0) { + patch.length1 += postcontext.length(); + patch.length2 += postcontext.length(); + if (!patch.diffs.isEmpty() + && patch.diffs.getLast().operation == Operation.EQUAL) { + patch.diffs.getLast().text += postcontext; + } else { + patch.diffs.add(new Diff(Operation.EQUAL, postcontext)); + } + } + if (!empty) { + pointer.add(patch); + } + } + bigpatch = pointer.hasNext() ? pointer.next() : null; + } + } + + /** + * Take a list of patches and return a textual representation. + * @param patches List of Patch objects. + * @return Text representation of patches. + */ + public String patch_toText(List patches) { + StringBuilder text = new StringBuilder(); + for (Patch aPatch : patches) { + text.append(aPatch); + } + return text.toString(); + } + + /** + * Parse a textual representation of patches and return a List of Patch + * objects. + * @param textline Text representation of patches. + * @return List of Patch objects. + * @throws IllegalArgumentException If invalid input. + */ + public List patch_fromText(String textline) + throws IllegalArgumentException { + List patches = new LinkedList(); + if (textline.length() == 0) { + return patches; + } + List textList = Arrays.asList(textline.split("\n")); + LinkedList text = new LinkedList(textList); + Patch patch; + Pattern patchHeader + = Pattern.compile("^@@ -(\\d+),?(\\d*) \\+(\\d+),?(\\d*) @@$"); + Matcher m; + char sign; + String line; + while (!text.isEmpty()) { + m = patchHeader.matcher(text.getFirst()); + if (!m.matches()) { + throw new IllegalArgumentException( + "Invalid patch string: " + text.getFirst()); + } + patch = new Patch(); + patches.add(patch); + patch.start1 = Integer.parseInt(m.group(1)); + if (m.group(2).length() == 0) { + patch.start1--; + patch.length1 = 1; + } else if (m.group(2).equals("0")) { + patch.length1 = 0; + } else { + patch.start1--; + patch.length1 = Integer.parseInt(m.group(2)); + } + + patch.start2 = Integer.parseInt(m.group(3)); + if (m.group(4).length() == 0) { + patch.start2--; + patch.length2 = 1; + } else if (m.group(4).equals("0")) { + patch.length2 = 0; + } else { + patch.start2--; + patch.length2 = Integer.parseInt(m.group(4)); + } + text.removeFirst(); + + while (!text.isEmpty()) { + try { + sign = text.getFirst().charAt(0); + } catch (IndexOutOfBoundsException e) { + // Blank line? Whatever. + text.removeFirst(); + continue; + } + line = text.getFirst().substring(1); + line = line.replace("+", "%2B"); // decode would change all "+" to " " + try { + line = URLDecoder.decode(line, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // Not likely on modern system. + throw new Error("This system does not support UTF-8.", e); + } catch (IllegalArgumentException e) { + // Malformed URI sequence. + throw new IllegalArgumentException( + "Illegal escape in patch_fromText: " + line, e); + } + if (sign == '-') { + // Deletion. + patch.diffs.add(new Diff(Operation.DELETE, line)); + } else if (sign == '+') { + // Insertion. + patch.diffs.add(new Diff(Operation.INSERT, line)); + } else if (sign == ' ') { + // Minor equality. + patch.diffs.add(new Diff(Operation.EQUAL, line)); + } else if (sign == '@') { + // Start of next patch. + break; + } else { + // WTF? + throw new IllegalArgumentException( + "Invalid patch mode '" + sign + "' in: " + line); + } + text.removeFirst(); + } + } + return patches; + } + + + /** + * Class representing one diff operation. + */ + public static class Diff { + /** + * One of: INSERT, DELETE or EQUAL. + */ + public Operation operation; + /** + * The text associated with this diff operation. + */ + public String text; + + /** + * Constructor. Initializes the diff with the provided values. + * @param operation One of INSERT, DELETE or EQUAL. + * @param text The text being applied. + */ + public Diff(Operation operation, String text) { + // Construct a diff with the specified operation and text. + this.operation = operation; + this.text = text; + } + + /** + * Display a human-readable version of this Diff. + * @return text version. + */ + public String toString() { + String prettyText = this.text.replace('\n', '\u00b6'); + return "Diff(" + this.operation + ",\"" + prettyText + "\")"; + } + + /** + * Create a numeric hash value for a Diff. + * This function is not used by DMP. + * @return Hash value. + */ + @Override + public int hashCode() { + final int prime = 31; + int result = (operation == null) ? 0 : operation.hashCode(); + result += prime * ((text == null) ? 0 : text.hashCode()); + return result; + } + + /** + * Is this Diff equivalent to another Diff? + * @param obj Another Diff to compare against. + * @return true or false. + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Diff other = (Diff) obj; + if (operation != other.operation) { + return false; + } + if (text == null) { + if (other.text != null) { + return false; + } + } else if (!text.equals(other.text)) { + return false; + } + return true; + } + } + + + /** + * Class representing one patch operation. + */ + public static class Patch { + public LinkedList diffs; + public int start1; + public int start2; + public int length1; + public int length2; + + /** + * Constructor. Initializes with an empty list of diffs. + */ + public Patch() { + this.diffs = new LinkedList(); + } + + /** + * Emulate GNU diff's format. + * Header: @@ -382,8 +481,9 @@ + * Indices are printed as 1-based, not 0-based. + * @return The GNU diff string. + */ + public String toString() { + String coords1, coords2; + if (this.length1 == 0) { + coords1 = this.start1 + ",0"; + } else if (this.length1 == 1) { + coords1 = Integer.toString(this.start1 + 1); + } else { + coords1 = (this.start1 + 1) + "," + this.length1; + } + if (this.length2 == 0) { + coords2 = this.start2 + ",0"; + } else if (this.length2 == 1) { + coords2 = Integer.toString(this.start2 + 1); + } else { + coords2 = (this.start2 + 1) + "," + this.length2; + } + StringBuilder text = new StringBuilder(); + text.append("@@ -").append(coords1).append(" +").append(coords2) + .append(" @@\n"); + // Escape the body of the patch with %xx notation. + for (Diff aDiff : this.diffs) { + switch (aDiff.operation) { + case INSERT: + text.append('+'); + break; + case DELETE: + text.append('-'); + break; + case EQUAL: + text.append(' '); + break; + } + try { + text.append(URLEncoder.encode(aDiff.text, "UTF-8").replace('+', ' ')) + .append("\n"); + } catch (UnsupportedEncodingException e) { + // Not likely on modern system. + throw new Error("This system does not support UTF-8.", e); + } + } + return unescapeForEncodeUriCompatability(text.toString()); + } + } + + /** + * Unescape selected chars for compatability with JavaScript's encodeURI. + * In speed critical applications this could be dropped since the + * receiving application will certainly decode these fine. + * Note that this function is case-sensitive. Thus "%3f" would not be + * unescaped. But this is ok because it is only called with the output of + * URLEncoder.encode which returns uppercase hex. + * + * Example: "%3F" -> "?", "%24" -> "$", etc. + * + * @param str The string to escape. + * @return The escaped string. + */ + private static String unescapeForEncodeUriCompatability(String str) { + return str.replace("%21", "!").replace("%7E", "~") + .replace("%27", "'").replace("%28", "(").replace("%29", ")") + .replace("%3B", ";").replace("%2F", "/").replace("%3F", "?") + .replace("%3A", ":").replace("%40", "@").replace("%26", "&") + .replace("%3D", "=").replace("%2B", "+").replace("%24", "$") + .replace("%2C", ",").replace("%23", "#"); + } +} diff --git a/dnet-dedup-test/src/test/resources/eu/dnetlib/pace/organization.test.conf b/dnet-dedup-test/src/test/resources/eu/dnetlib/pace/organization.test.conf new file mode 100644 index 0000000..3137253 --- /dev/null +++ b/dnet-dedup-test/src/test/resources/eu/dnetlib/pace/organization.test.conf @@ -0,0 +1,27 @@ +{ + "wf" : { + "threshold" : "0.98", + "dedupRun" : "001", + "entityType" : "organization", + "orderField" : "legalname", + "queueMaxSize" : "2000", + "groupMaxSize" : "10", + "slidingWindowSize" : "200", + "rootBuilder" : [ "organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy" ], + "includeChildren" : "true" + }, + "pace" : { + "clustering" : [ + { "name" : "ngrampairs", "fields" : [ "legalname" ], "params" : { "max" : "1", "ngramLen" : "3"} }, + { "name" : "suffixprefix", "fields" : [ "legalname" ], "params" : { "max" : "1", "len" : "3" } } + ], + "conditions" : [ + { "name" : "exactMatch", "fields" : [ "country" ] } + ], + "model" : [ + { "name" : "legalname", "algo" : "JaroWinkler", "type" : "String", "weight" : "1.0", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value" }, + { "name" : "country", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/country/classid" } + ], + "blacklists" : { } + } +} \ No newline at end of file diff --git a/dnet-dedup-test/src/test/resources/eu/dnetlib/pace/result.simple.pace.conf b/dnet-dedup-test/src/test/resources/eu/dnetlib/pace/result.simple.pace.conf index 910fbcd..5fa9b84 100644 --- a/dnet-dedup-test/src/test/resources/eu/dnetlib/pace/result.simple.pace.conf +++ b/dnet-dedup-test/src/test/resources/eu/dnetlib/pace/result.simple.pace.conf @@ -1,7 +1,7 @@ { "wf" : { "threshold" : "0.99", - "run" : "001", + "dedupRun" : "001", "entityType" : "result", "orderField" : "title", "queueMaxSize" : "2000", diff --git a/dnet-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java b/dnet-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java new file mode 100644 index 0000000..fbbc9d7 --- /dev/null +++ b/dnet-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java @@ -0,0 +1,2471 @@ +package eu.dnetlib.pace.util; + +/* + * Diff Match and Patch + * Copyright 2018 The diff-match-patch Authors. + * https://github.com/google/diff-match-patch + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/* + * Functions for diff, match and patch. + * Computes the difference between two texts to create a patch. + * Applies the patch onto another text, allowing for errors. + * + * @author fraser@google.com (Neil Fraser) + */ + +/** + * Class containing the diff, match and patch methods. + * Also contains the behaviour settings. + */ +public class DiffPatchMatch { + + // Defaults. + // Set these on your diff_match_patch instance to override the defaults. + + /** + * Number of seconds to map a diff before giving up (0 for infinity). + */ + public float Diff_Timeout = 1.0f; + /** + * Cost of an empty edit operation in terms of edit characters. + */ + public short Diff_EditCost = 4; + /** + * At what point is no match declared (0.0 = perfection, 1.0 = very loose). + */ + public float Match_Threshold = 0.5f; + /** + * How far to search for a match (0 = exact location, 1000+ = broad match). + * A match this many characters away from the expected location will add + * 1.0 to the score (0.0 is a perfect match). + */ + public int Match_Distance = 1000; + /** + * When deleting a large block of text (over ~64 characters), how close do + * the contents have to be to match the expected contents. (0.0 = perfection, + * 1.0 = very loose). Note that Match_Threshold controls how closely the + * end points of a delete need to match. + */ + public float Patch_DeleteThreshold = 0.5f; + /** + * Chunk size for context length. + */ + public short Patch_Margin = 4; + + /** + * The number of bits in an int. + */ + private short Match_MaxBits = 32; + + /** + * Internal class for returning results from diff_linesToChars(). + * Other less paranoid languages just use a three-element array. + */ + protected static class LinesToCharsResult { + protected String chars1; + protected String chars2; + protected List lineArray; + + protected LinesToCharsResult(String chars1, String chars2, + List lineArray) { + this.chars1 = chars1; + this.chars2 = chars2; + this.lineArray = lineArray; + } + } + + + // DIFF FUNCTIONS + + + /** + * The data structure representing a diff is a Linked list of Diff objects: + * {Diff(Operation.DELETE, "Hello"), Diff(Operation.INSERT, "Goodbye"), + * Diff(Operation.EQUAL, " world.")} + * which means: delete "Hello", add "Goodbye" and keep " world." + */ + public enum Operation { + DELETE, INSERT, EQUAL + } + + /** + * Find the differences between two texts. + * Run a faster, slightly less optimal diff. + * This method allows the 'checklines' of diff_main() to be optional. + * Most of the time checklines is wanted, so default to true. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @return Linked List of Diff objects. + */ + public LinkedList diff_main(String text1, String text2) { + return diff_main(text1, text2, true); + } + + /** + * Find the differences between two texts. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param checklines Speedup flag. If false, then don't run a + * line-level diff first to identify the changed areas. + * If true, then run a faster slightly less optimal diff. + * @return Linked List of Diff objects. + */ + public LinkedList diff_main(String text1, String text2, + boolean checklines) { + // Set a deadline by which time the diff must be complete. + long deadline; + if (Diff_Timeout <= 0) { + deadline = Long.MAX_VALUE; + } else { + deadline = System.currentTimeMillis() + (long) (Diff_Timeout * 1000); + } + return diff_main(text1, text2, checklines, deadline); + } + + /** + * Find the differences between two texts. Simplifies the problem by + * stripping any common prefix or suffix off the texts before diffing. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param checklines Speedup flag. If false, then don't run a + * line-level diff first to identify the changed areas. + * If true, then run a faster slightly less optimal diff. + * @param deadline Time when the diff should be complete by. Used + * internally for recursive calls. Users should set DiffTimeout instead. + * @return Linked List of Diff objects. + */ + private LinkedList diff_main(String text1, String text2, + boolean checklines, long deadline) { + // Check for null inputs. + if (text1 == null || text2 == null) { + throw new IllegalArgumentException("Null inputs. (diff_main)"); + } + + // Check for equality (speedup). + LinkedList diffs; + if (text1.equals(text2)) { + diffs = new LinkedList(); + if (text1.length() != 0) { + diffs.add(new Diff(Operation.EQUAL, text1)); + } + return diffs; + } + + // Trim off common prefix (speedup). + int commonlength = diff_commonPrefix(text1, text2); + String commonprefix = text1.substring(0, commonlength); + text1 = text1.substring(commonlength); + text2 = text2.substring(commonlength); + + // Trim off common suffix (speedup). + commonlength = diff_commonSuffix(text1, text2); + String commonsuffix = text1.substring(text1.length() - commonlength); + text1 = text1.substring(0, text1.length() - commonlength); + text2 = text2.substring(0, text2.length() - commonlength); + + // Compute the diff on the middle block. + diffs = diff_compute(text1, text2, checklines, deadline); + + // Restore the prefix and suffix. + if (commonprefix.length() != 0) { + diffs.addFirst(new Diff(Operation.EQUAL, commonprefix)); + } + if (commonsuffix.length() != 0) { + diffs.addLast(new Diff(Operation.EQUAL, commonsuffix)); + } + + diff_cleanupMerge(diffs); + return diffs; + } + + /** + * Find the differences between two texts. Assumes that the texts do not + * have any common prefix or suffix. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param checklines Speedup flag. If false, then don't run a + * line-level diff first to identify the changed areas. + * If true, then run a faster slightly less optimal diff. + * @param deadline Time when the diff should be complete by. + * @return Linked List of Diff objects. + */ + private LinkedList diff_compute(String text1, String text2, + boolean checklines, long deadline) { + LinkedList diffs = new LinkedList(); + + if (text1.length() == 0) { + // Just add some text (speedup). + diffs.add(new Diff(Operation.INSERT, text2)); + return diffs; + } + + if (text2.length() == 0) { + // Just delete some text (speedup). + diffs.add(new Diff(Operation.DELETE, text1)); + return diffs; + } + + String longtext = text1.length() > text2.length() ? text1 : text2; + String shorttext = text1.length() > text2.length() ? text2 : text1; + int i = longtext.indexOf(shorttext); + if (i != -1) { + // Shorter text is inside the longer text (speedup). + Operation op = (text1.length() > text2.length()) ? + Operation.DELETE : Operation.INSERT; + diffs.add(new Diff(op, longtext.substring(0, i))); + diffs.add(new Diff(Operation.EQUAL, shorttext)); + diffs.add(new Diff(op, longtext.substring(i + shorttext.length()))); + return diffs; + } + + if (shorttext.length() == 1) { + // Single character string. + // After the previous speedup, the character can't be an equality. + diffs.add(new Diff(Operation.DELETE, text1)); + diffs.add(new Diff(Operation.INSERT, text2)); + return diffs; + } + + // Check to see if the problem can be split in two. + String[] hm = diff_halfMatch(text1, text2); + if (hm != null) { + // A half-match was found, sort out the return data. + String text1_a = hm[0]; + String text1_b = hm[1]; + String text2_a = hm[2]; + String text2_b = hm[3]; + String mid_common = hm[4]; + // Send both pairs off for separate processing. + LinkedList diffs_a = diff_main(text1_a, text2_a, + checklines, deadline); + LinkedList diffs_b = diff_main(text1_b, text2_b, + checklines, deadline); + // Merge the results. + diffs = diffs_a; + diffs.add(new Diff(Operation.EQUAL, mid_common)); + diffs.addAll(diffs_b); + return diffs; + } + + if (checklines && text1.length() > 100 && text2.length() > 100) { + return diff_lineMode(text1, text2, deadline); + } + + return diff_bisect(text1, text2, deadline); + } + + /** + * Do a quick line-level diff on both strings, then rediff the parts for + * greater accuracy. + * This speedup can produce non-minimal diffs. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param deadline Time when the diff should be complete by. + * @return Linked List of Diff objects. + */ + private LinkedList diff_lineMode(String text1, String text2, + long deadline) { + // Scan the text on a line-by-line basis first. + LinesToCharsResult a = diff_linesToChars(text1, text2); + text1 = a.chars1; + text2 = a.chars2; + List linearray = a.lineArray; + + LinkedList diffs = diff_main(text1, text2, false, deadline); + + // Convert the diff back to original text. + diff_charsToLines(diffs, linearray); + // Eliminate freak matches (e.g. blank lines) + diff_cleanupSemantic(diffs); + + // Rediff any replacement blocks, this time character-by-character. + // Add a dummy entry at the end. + diffs.add(new Diff(Operation.EQUAL, "")); + int count_delete = 0; + int count_insert = 0; + String text_delete = ""; + String text_insert = ""; + ListIterator pointer = diffs.listIterator(); + Diff thisDiff = pointer.next(); + while (thisDiff != null) { + switch (thisDiff.operation) { + case INSERT: + count_insert++; + text_insert += thisDiff.text; + break; + case DELETE: + count_delete++; + text_delete += thisDiff.text; + break; + case EQUAL: + // Upon reaching an equality, check for prior redundancies. + if (count_delete >= 1 && count_insert >= 1) { + // Delete the offending records and add the merged ones. + pointer.previous(); + for (int j = 0; j < count_delete + count_insert; j++) { + pointer.previous(); + pointer.remove(); + } + for (Diff subDiff : diff_main(text_delete, text_insert, false, + deadline)) { + pointer.add(subDiff); + } + } + count_insert = 0; + count_delete = 0; + text_delete = ""; + text_insert = ""; + break; + } + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + diffs.removeLast(); // Remove the dummy entry at the end. + + return diffs; + } + + /** + * Find the 'middle snake' of a diff, split the problem in two + * and return the recursively constructed diff. + * See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param deadline Time at which to bail if not yet complete. + * @return LinkedList of Diff objects. + */ + protected LinkedList diff_bisect(String text1, String text2, + long deadline) { + // Cache the text lengths to prevent multiple calls. + int text1_length = text1.length(); + int text2_length = text2.length(); + int max_d = (text1_length + text2_length + 1) / 2; + int v_offset = max_d; + int v_length = 2 * max_d; + int[] v1 = new int[v_length]; + int[] v2 = new int[v_length]; + for (int x = 0; x < v_length; x++) { + v1[x] = -1; + v2[x] = -1; + } + v1[v_offset + 1] = 0; + v2[v_offset + 1] = 0; + int delta = text1_length - text2_length; + // If the total number of characters is odd, then the front path will + // collide with the reverse path. + boolean front = (delta % 2 != 0); + // Offsets for start and end of k loop. + // Prevents mapping of space beyond the grid. + int k1start = 0; + int k1end = 0; + int k2start = 0; + int k2end = 0; + for (int d = 0; d < max_d; d++) { + // Bail out if deadline is reached. + if (System.currentTimeMillis() > deadline) { + break; + } + + // Walk the front path one step. + for (int k1 = -d + k1start; k1 <= d - k1end; k1 += 2) { + int k1_offset = v_offset + k1; + int x1; + if (k1 == -d || (k1 != d && v1[k1_offset - 1] < v1[k1_offset + 1])) { + x1 = v1[k1_offset + 1]; + } else { + x1 = v1[k1_offset - 1] + 1; + } + int y1 = x1 - k1; + while (x1 < text1_length && y1 < text2_length + && text1.charAt(x1) == text2.charAt(y1)) { + x1++; + y1++; + } + v1[k1_offset] = x1; + if (x1 > text1_length) { + // Ran off the right of the graph. + k1end += 2; + } else if (y1 > text2_length) { + // Ran off the bottom of the graph. + k1start += 2; + } else if (front) { + int k2_offset = v_offset + delta - k1; + if (k2_offset >= 0 && k2_offset < v_length && v2[k2_offset] != -1) { + // Mirror x2 onto top-left coordinate system. + int x2 = text1_length - v2[k2_offset]; + if (x1 >= x2) { + // Overlap detected. + return diff_bisectSplit(text1, text2, x1, y1, deadline); + } + } + } + } + + // Walk the reverse path one step. + for (int k2 = -d + k2start; k2 <= d - k2end; k2 += 2) { + int k2_offset = v_offset + k2; + int x2; + if (k2 == -d || (k2 != d && v2[k2_offset - 1] < v2[k2_offset + 1])) { + x2 = v2[k2_offset + 1]; + } else { + x2 = v2[k2_offset - 1] + 1; + } + int y2 = x2 - k2; + while (x2 < text1_length && y2 < text2_length + && text1.charAt(text1_length - x2 - 1) + == text2.charAt(text2_length - y2 - 1)) { + x2++; + y2++; + } + v2[k2_offset] = x2; + if (x2 > text1_length) { + // Ran off the left of the graph. + k2end += 2; + } else if (y2 > text2_length) { + // Ran off the top of the graph. + k2start += 2; + } else if (!front) { + int k1_offset = v_offset + delta - k2; + if (k1_offset >= 0 && k1_offset < v_length && v1[k1_offset] != -1) { + int x1 = v1[k1_offset]; + int y1 = v_offset + x1 - k1_offset; + // Mirror x2 onto top-left coordinate system. + x2 = text1_length - x2; + if (x1 >= x2) { + // Overlap detected. + return diff_bisectSplit(text1, text2, x1, y1, deadline); + } + } + } + } + } + // Diff took too long and hit the deadline or + // number of diffs equals number of characters, no commonality at all. + LinkedList diffs = new LinkedList(); + diffs.add(new Diff(Operation.DELETE, text1)); + diffs.add(new Diff(Operation.INSERT, text2)); + return diffs; + } + + /** + * Given the location of the 'middle snake', split the diff in two parts + * and recurse. + * @param text1 Old string to be diffed. + * @param text2 New string to be diffed. + * @param x Index of split point in text1. + * @param y Index of split point in text2. + * @param deadline Time at which to bail if not yet complete. + * @return LinkedList of Diff objects. + */ + private LinkedList diff_bisectSplit(String text1, String text2, + int x, int y, long deadline) { + String text1a = text1.substring(0, x); + String text2a = text2.substring(0, y); + String text1b = text1.substring(x); + String text2b = text2.substring(y); + + // Compute both diffs serially. + LinkedList diffs = diff_main(text1a, text2a, false, deadline); + LinkedList diffsb = diff_main(text1b, text2b, false, deadline); + + diffs.addAll(diffsb); + return diffs; + } + + /** + * Split two texts into a list of strings. Reduce the texts to a string of + * hashes where each Unicode character represents one line. + * @param text1 First string. + * @param text2 Second string. + * @return An object containing the encoded text1, the encoded text2 and + * the List of unique strings. The zeroth element of the List of + * unique strings is intentionally blank. + */ + protected LinesToCharsResult diff_linesToChars(String text1, String text2) { + List lineArray = new ArrayList(); + Map lineHash = new HashMap(); + // e.g. linearray[4] == "Hello\n" + // e.g. linehash.get("Hello\n") == 4 + + // "\x00" is a valid character, but various debuggers don't like it. + // So we'll insert a junk entry to avoid generating a null character. + lineArray.add(""); + + // Allocate 2/3rds of the space for text1, the rest for text2. + String chars1 = diff_linesToCharsMunge(text1, lineArray, lineHash, 40000); + String chars2 = diff_linesToCharsMunge(text2, lineArray, lineHash, 65535); + return new LinesToCharsResult(chars1, chars2, lineArray); + } + + /** + * Split a text into a list of strings. Reduce the texts to a string of + * hashes where each Unicode character represents one line. + * @param text String to encode. + * @param lineArray List of unique strings. + * @param lineHash Map of strings to indices. + * @param maxLines Maximum length of lineArray. + * @return Encoded string. + */ + private String diff_linesToCharsMunge(String text, List lineArray, + Map lineHash, int maxLines) { + int lineStart = 0; + int lineEnd = -1; + String line; + StringBuilder chars = new StringBuilder(); + // Walk the text, pulling out a substring for each line. + // text.split('\n') would would temporarily double our memory footprint. + // Modifying text would create many large strings to garbage collect. + while (lineEnd < text.length() - 1) { + lineEnd = text.indexOf('\n', lineStart); + if (lineEnd == -1) { + lineEnd = text.length() - 1; + } + line = text.substring(lineStart, lineEnd + 1); + + if (lineHash.containsKey(line)) { + chars.append(String.valueOf((char) (int) lineHash.get(line))); + } else { + if (lineArray.size() == maxLines) { + // Bail out at 65535 because + // String.valueOf((char) 65536).equals(String.valueOf(((char) 0))) + line = text.substring(lineStart); + lineEnd = text.length(); + } + lineArray.add(line); + lineHash.put(line, lineArray.size() - 1); + chars.append(String.valueOf((char) (lineArray.size() - 1))); + } + lineStart = lineEnd + 1; + } + return chars.toString(); + } + + /** + * Rehydrate the text in a diff from a string of line hashes to real lines of + * text. + * @param diffs List of Diff objects. + * @param lineArray List of unique strings. + */ + protected void diff_charsToLines(List diffs, + List lineArray) { + StringBuilder text; + for (Diff diff : diffs) { + text = new StringBuilder(); + for (int j = 0; j < diff.text.length(); j++) { + text.append(lineArray.get(diff.text.charAt(j))); + } + diff.text = text.toString(); + } + } + + /** + * Determine the common prefix of two strings + * @param text1 First string. + * @param text2 Second string. + * @return The number of characters common to the start of each string. + */ + public int diff_commonPrefix(String text1, String text2) { + // Performance analysis: https://neil.fraser.name/news/2007/10/09/ + int n = Math.min(text1.length(), text2.length()); + for (int i = 0; i < n; i++) { + if (text1.charAt(i) != text2.charAt(i)) { + return i; + } + } + return n; + } + + /** + * Determine the common suffix of two strings + * @param text1 First string. + * @param text2 Second string. + * @return The number of characters common to the end of each string. + */ + public int diff_commonSuffix(String text1, String text2) { + // Performance analysis: https://neil.fraser.name/news/2007/10/09/ + int text1_length = text1.length(); + int text2_length = text2.length(); + int n = Math.min(text1_length, text2_length); + for (int i = 1; i <= n; i++) { + if (text1.charAt(text1_length - i) != text2.charAt(text2_length - i)) { + return i - 1; + } + } + return n; + } + + /** + * Determine if the suffix of one string is the prefix of another. + * @param text1 First string. + * @param text2 Second string. + * @return The number of characters common to the end of the first + * string and the start of the second string. + */ + protected int diff_commonOverlap(String text1, String text2) { + // Cache the text lengths to prevent multiple calls. + int text1_length = text1.length(); + int text2_length = text2.length(); + // Eliminate the null case. + if (text1_length == 0 || text2_length == 0) { + return 0; + } + // Truncate the longer string. + if (text1_length > text2_length) { + text1 = text1.substring(text1_length - text2_length); + } else if (text1_length < text2_length) { + text2 = text2.substring(0, text1_length); + } + int text_length = Math.min(text1_length, text2_length); + // Quick check for the worst case. + if (text1.equals(text2)) { + return text_length; + } + + // Start by looking for a single character match + // and increase length until no match is found. + // Performance analysis: https://neil.fraser.name/news/2010/11/04/ + int best = 0; + int length = 1; + while (true) { + String pattern = text1.substring(text_length - length); + int found = text2.indexOf(pattern); + if (found == -1) { + return best; + } + length += found; + if (found == 0 || text1.substring(text_length - length).equals( + text2.substring(0, length))) { + best = length; + length++; + } + } + } + + /** + * Do the two texts share a substring which is at least half the length of + * the longer text? + * This speedup can produce non-minimal diffs. + * @param text1 First string. + * @param text2 Second string. + * @return Five element String array, containing the prefix of text1, the + * suffix of text1, the prefix of text2, the suffix of text2 and the + * common middle. Or null if there was no match. + */ + protected String[] diff_halfMatch(String text1, String text2) { + if (Diff_Timeout <= 0) { + // Don't risk returning a non-optimal diff if we have unlimited time. + return null; + } + String longtext = text1.length() > text2.length() ? text1 : text2; + String shorttext = text1.length() > text2.length() ? text2 : text1; + if (longtext.length() < 4 || shorttext.length() * 2 < longtext.length()) { + return null; // Pointless. + } + + // First check if the second quarter is the seed for a half-match. + String[] hm1 = diff_halfMatchI(longtext, shorttext, + (longtext.length() + 3) / 4); + // Check again based on the third quarter. + String[] hm2 = diff_halfMatchI(longtext, shorttext, + (longtext.length() + 1) / 2); + String[] hm; + if (hm1 == null && hm2 == null) { + return null; + } else if (hm2 == null) { + hm = hm1; + } else if (hm1 == null) { + hm = hm2; + } else { + // Both matched. Select the longest. + hm = hm1[4].length() > hm2[4].length() ? hm1 : hm2; + } + + // A half-match was found, sort out the return data. + if (text1.length() > text2.length()) { + return hm; + //return new String[]{hm[0], hm[1], hm[2], hm[3], hm[4]}; + } else { + return new String[]{hm[2], hm[3], hm[0], hm[1], hm[4]}; + } + } + + /** + * Does a substring of shorttext exist within longtext such that the + * substring is at least half the length of longtext? + * @param longtext Longer string. + * @param shorttext Shorter string. + * @param i Start index of quarter length substring within longtext. + * @return Five element String array, containing the prefix of longtext, the + * suffix of longtext, the prefix of shorttext, the suffix of shorttext + * and the common middle. Or null if there was no match. + */ + private String[] diff_halfMatchI(String longtext, String shorttext, int i) { + // Start with a 1/4 length substring at position i as a seed. + String seed = longtext.substring(i, i + longtext.length() / 4); + int j = -1; + String best_common = ""; + String best_longtext_a = "", best_longtext_b = ""; + String best_shorttext_a = "", best_shorttext_b = ""; + while ((j = shorttext.indexOf(seed, j + 1)) != -1) { + int prefixLength = diff_commonPrefix(longtext.substring(i), + shorttext.substring(j)); + int suffixLength = diff_commonSuffix(longtext.substring(0, i), + shorttext.substring(0, j)); + if (best_common.length() < suffixLength + prefixLength) { + best_common = shorttext.substring(j - suffixLength, j) + + shorttext.substring(j, j + prefixLength); + best_longtext_a = longtext.substring(0, i - suffixLength); + best_longtext_b = longtext.substring(i + prefixLength); + best_shorttext_a = shorttext.substring(0, j - suffixLength); + best_shorttext_b = shorttext.substring(j + prefixLength); + } + } + if (best_common.length() * 2 >= longtext.length()) { + return new String[]{best_longtext_a, best_longtext_b, + best_shorttext_a, best_shorttext_b, best_common}; + } else { + return null; + } + } + + /** + * Reduce the number of edits by eliminating semantically trivial equalities. + * @param diffs LinkedList of Diff objects. + */ + public void diff_cleanupSemantic(LinkedList diffs) { + if (diffs.isEmpty()) { + return; + } + boolean changes = false; + Deque equalities = new ArrayDeque(); // Double-ended queue of qualities. + String lastEquality = null; // Always equal to equalities.peek().text + ListIterator pointer = diffs.listIterator(); + // Number of characters that changed prior to the equality. + int length_insertions1 = 0; + int length_deletions1 = 0; + // Number of characters that changed after the equality. + int length_insertions2 = 0; + int length_deletions2 = 0; + Diff thisDiff = pointer.next(); + while (thisDiff != null) { + if (thisDiff.operation == Operation.EQUAL) { + // Equality found. + equalities.push(thisDiff); + length_insertions1 = length_insertions2; + length_deletions1 = length_deletions2; + length_insertions2 = 0; + length_deletions2 = 0; + lastEquality = thisDiff.text; + } else { + // An insertion or deletion. + if (thisDiff.operation == Operation.INSERT) { + length_insertions2 += thisDiff.text.length(); + } else { + length_deletions2 += thisDiff.text.length(); + } + // Eliminate an equality that is smaller or equal to the edits on both + // sides of it. + if (lastEquality != null && (lastEquality.length() + <= Math.max(length_insertions1, length_deletions1)) + && (lastEquality.length() + <= Math.max(length_insertions2, length_deletions2))) { + //System.out.println("Splitting: '" + lastEquality + "'"); + // Walk back to offending equality. + while (thisDiff != equalities.peek()) { + thisDiff = pointer.previous(); + } + pointer.next(); + + // Replace equality with a delete. + pointer.set(new Diff(Operation.DELETE, lastEquality)); + // Insert a corresponding an insert. + pointer.add(new Diff(Operation.INSERT, lastEquality)); + + equalities.pop(); // Throw away the equality we just deleted. + if (!equalities.isEmpty()) { + // Throw away the previous equality (it needs to be reevaluated). + equalities.pop(); + } + if (equalities.isEmpty()) { + // There are no previous equalities, walk back to the start. + while (pointer.hasPrevious()) { + pointer.previous(); + } + } else { + // There is a safe equality we can fall back to. + thisDiff = equalities.peek(); + while (thisDiff != pointer.previous()) { + // Intentionally empty loop. + } + } + + length_insertions1 = 0; // Reset the counters. + length_insertions2 = 0; + length_deletions1 = 0; + length_deletions2 = 0; + lastEquality = null; + changes = true; + } + } + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + + // Normalize the diff. + if (changes) { + diff_cleanupMerge(diffs); + } + diff_cleanupSemanticLossless(diffs); + + // Find any overlaps between deletions and insertions. + // e.g: abcxxxxxxdef + // -> abcxxxdef + // e.g: xxxabcdefxxx + // -> defxxxabc + // Only extract an overlap if it is as big as the edit ahead or behind it. + pointer = diffs.listIterator(); + Diff prevDiff = null; + thisDiff = null; + if (pointer.hasNext()) { + prevDiff = pointer.next(); + if (pointer.hasNext()) { + thisDiff = pointer.next(); + } + } + while (thisDiff != null) { + if (prevDiff.operation == Operation.DELETE && + thisDiff.operation == Operation.INSERT) { + String deletion = prevDiff.text; + String insertion = thisDiff.text; + int overlap_length1 = this.diff_commonOverlap(deletion, insertion); + int overlap_length2 = this.diff_commonOverlap(insertion, deletion); + if (overlap_length1 >= overlap_length2) { + if (overlap_length1 >= deletion.length() / 2.0 || + overlap_length1 >= insertion.length() / 2.0) { + // Overlap found. Insert an equality and trim the surrounding edits. + pointer.previous(); + pointer.add(new Diff(Operation.EQUAL, + insertion.substring(0, overlap_length1))); + prevDiff.text = + deletion.substring(0, deletion.length() - overlap_length1); + thisDiff.text = insertion.substring(overlap_length1); + // pointer.add inserts the element before the cursor, so there is + // no need to step past the new element. + } + } else { + if (overlap_length2 >= deletion.length() / 2.0 || + overlap_length2 >= insertion.length() / 2.0) { + // Reverse overlap found. + // Insert an equality and swap and trim the surrounding edits. + pointer.previous(); + pointer.add(new Diff(Operation.EQUAL, + deletion.substring(0, overlap_length2))); + prevDiff.operation = Operation.INSERT; + prevDiff.text = + insertion.substring(0, insertion.length() - overlap_length2); + thisDiff.operation = Operation.DELETE; + thisDiff.text = deletion.substring(overlap_length2); + // pointer.add inserts the element before the cursor, so there is + // no need to step past the new element. + } + } + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + prevDiff = thisDiff; + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + } + + /** + * Look for single edits surrounded on both sides by equalities + * which can be shifted sideways to align the edit to a word boundary. + * e.g: The cat came. -> The cat came. + * @param diffs LinkedList of Diff objects. + */ + public void diff_cleanupSemanticLossless(LinkedList diffs) { + String equality1, edit, equality2; + String commonString; + int commonOffset; + int score, bestScore; + String bestEquality1, bestEdit, bestEquality2; + // Create a new iterator at the start. + ListIterator pointer = diffs.listIterator(); + Diff prevDiff = pointer.hasNext() ? pointer.next() : null; + Diff thisDiff = pointer.hasNext() ? pointer.next() : null; + Diff nextDiff = pointer.hasNext() ? pointer.next() : null; + // Intentionally ignore the first and last element (don't need checking). + while (nextDiff != null) { + if (prevDiff.operation == Operation.EQUAL && + nextDiff.operation == Operation.EQUAL) { + // This is a single edit surrounded by equalities. + equality1 = prevDiff.text; + edit = thisDiff.text; + equality2 = nextDiff.text; + + // First, shift the edit as far left as possible. + commonOffset = diff_commonSuffix(equality1, edit); + if (commonOffset != 0) { + commonString = edit.substring(edit.length() - commonOffset); + equality1 = equality1.substring(0, equality1.length() - commonOffset); + edit = commonString + edit.substring(0, edit.length() - commonOffset); + equality2 = commonString + equality2; + } + + // Second, step character by character right, looking for the best fit. + bestEquality1 = equality1; + bestEdit = edit; + bestEquality2 = equality2; + bestScore = diff_cleanupSemanticScore(equality1, edit) + + diff_cleanupSemanticScore(edit, equality2); + while (edit.length() != 0 && equality2.length() != 0 + && edit.charAt(0) == equality2.charAt(0)) { + equality1 += edit.charAt(0); + edit = edit.substring(1) + equality2.charAt(0); + equality2 = equality2.substring(1); + score = diff_cleanupSemanticScore(equality1, edit) + + diff_cleanupSemanticScore(edit, equality2); + // The >= encourages trailing rather than leading whitespace on edits. + if (score >= bestScore) { + bestScore = score; + bestEquality1 = equality1; + bestEdit = edit; + bestEquality2 = equality2; + } + } + + if (!prevDiff.text.equals(bestEquality1)) { + // We have an improvement, save it back to the diff. + if (bestEquality1.length() != 0) { + prevDiff.text = bestEquality1; + } else { + pointer.previous(); // Walk past nextDiff. + pointer.previous(); // Walk past thisDiff. + pointer.previous(); // Walk past prevDiff. + pointer.remove(); // Delete prevDiff. + pointer.next(); // Walk past thisDiff. + pointer.next(); // Walk past nextDiff. + } + thisDiff.text = bestEdit; + if (bestEquality2.length() != 0) { + nextDiff.text = bestEquality2; + } else { + pointer.remove(); // Delete nextDiff. + nextDiff = thisDiff; + thisDiff = prevDiff; + } + } + } + prevDiff = thisDiff; + thisDiff = nextDiff; + nextDiff = pointer.hasNext() ? pointer.next() : null; + } + } + + /** + * Given two strings, compute a score representing whether the internal + * boundary falls on logical boundaries. + * Scores range from 6 (best) to 0 (worst). + * @param one First string. + * @param two Second string. + * @return The score. + */ + private int diff_cleanupSemanticScore(String one, String two) { + if (one.length() == 0 || two.length() == 0) { + // Edges are the best. + return 6; + } + + // Each port of this function behaves slightly differently due to + // subtle differences in each language's definition of things like + // 'whitespace'. Since this function's purpose is largely cosmetic, + // the choice has been made to use each language's native features + // rather than force total conformity. + char char1 = one.charAt(one.length() - 1); + char char2 = two.charAt(0); + boolean nonAlphaNumeric1 = !Character.isLetterOrDigit(char1); + boolean nonAlphaNumeric2 = !Character.isLetterOrDigit(char2); + boolean whitespace1 = nonAlphaNumeric1 && Character.isWhitespace(char1); + boolean whitespace2 = nonAlphaNumeric2 && Character.isWhitespace(char2); + boolean lineBreak1 = whitespace1 + && Character.getType(char1) == Character.CONTROL; + boolean lineBreak2 = whitespace2 + && Character.getType(char2) == Character.CONTROL; + boolean blankLine1 = lineBreak1 && BLANKLINEEND.matcher(one).find(); + boolean blankLine2 = lineBreak2 && BLANKLINESTART.matcher(two).find(); + + if (blankLine1 || blankLine2) { + // Five points for blank lines. + return 5; + } else if (lineBreak1 || lineBreak2) { + // Four points for line breaks. + return 4; + } else if (nonAlphaNumeric1 && !whitespace1 && whitespace2) { + // Three points for end of sentences. + return 3; + } else if (whitespace1 || whitespace2) { + // Two points for whitespace. + return 2; + } else if (nonAlphaNumeric1 || nonAlphaNumeric2) { + // One point for non-alphanumeric. + return 1; + } + return 0; + } + + // Define some regex patterns for matching boundaries. + private Pattern BLANKLINEEND + = Pattern.compile("\\n\\r?\\n\\Z", Pattern.DOTALL); + private Pattern BLANKLINESTART + = Pattern.compile("\\A\\r?\\n\\r?\\n", Pattern.DOTALL); + + /** + * Reduce the number of edits by eliminating operationally trivial equalities. + * @param diffs LinkedList of Diff objects. + */ + public void diff_cleanupEfficiency(LinkedList diffs) { + if (diffs.isEmpty()) { + return; + } + boolean changes = false; + Deque equalities = new ArrayDeque(); // Double-ended queue of equalities. + String lastEquality = null; // Always equal to equalities.peek().text + ListIterator pointer = diffs.listIterator(); + // Is there an insertion operation before the last equality. + boolean pre_ins = false; + // Is there a deletion operation before the last equality. + boolean pre_del = false; + // Is there an insertion operation after the last equality. + boolean post_ins = false; + // Is there a deletion operation after the last equality. + boolean post_del = false; + Diff thisDiff = pointer.next(); + Diff safeDiff = thisDiff; // The last Diff that is known to be unsplittable. + while (thisDiff != null) { + if (thisDiff.operation == Operation.EQUAL) { + // Equality found. + if (thisDiff.text.length() < Diff_EditCost && (post_ins || post_del)) { + // Candidate found. + equalities.push(thisDiff); + pre_ins = post_ins; + pre_del = post_del; + lastEquality = thisDiff.text; + } else { + // Not a candidate, and can never become one. + equalities.clear(); + lastEquality = null; + safeDiff = thisDiff; + } + post_ins = post_del = false; + } else { + // An insertion or deletion. + if (thisDiff.operation == Operation.DELETE) { + post_del = true; + } else { + post_ins = true; + } + /* + * Five types to be split: + * ABXYCD + * AXCD + * ABXC + * AXCD + * ABXC + */ + if (lastEquality != null + && ((pre_ins && pre_del && post_ins && post_del) + || ((lastEquality.length() < Diff_EditCost / 2) + && ((pre_ins ? 1 : 0) + (pre_del ? 1 : 0) + + (post_ins ? 1 : 0) + (post_del ? 1 : 0)) == 3))) { + //System.out.println("Splitting: '" + lastEquality + "'"); + // Walk back to offending equality. + while (thisDiff != equalities.peek()) { + thisDiff = pointer.previous(); + } + pointer.next(); + + // Replace equality with a delete. + pointer.set(new Diff(Operation.DELETE, lastEquality)); + // Insert a corresponding an insert. + pointer.add(thisDiff = new Diff(Operation.INSERT, lastEquality)); + + equalities.pop(); // Throw away the equality we just deleted. + lastEquality = null; + if (pre_ins && pre_del) { + // No changes made which could affect previous entry, keep going. + post_ins = post_del = true; + equalities.clear(); + safeDiff = thisDiff; + } else { + if (!equalities.isEmpty()) { + // Throw away the previous equality (it needs to be reevaluated). + equalities.pop(); + } + if (equalities.isEmpty()) { + // There are no previous questionable equalities, + // walk back to the last known safe diff. + thisDiff = safeDiff; + } else { + // There is an equality we can fall back to. + thisDiff = equalities.peek(); + } + while (thisDiff != pointer.previous()) { + // Intentionally empty loop. + } + post_ins = post_del = false; + } + + changes = true; + } + } + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + + if (changes) { + diff_cleanupMerge(diffs); + } + } + + /** + * Reorder and merge like edit sections. Merge equalities. + * Any edit section can move as long as it doesn't cross an equality. + * @param diffs LinkedList of Diff objects. + */ + public void diff_cleanupMerge(LinkedList diffs) { + diffs.add(new Diff(Operation.EQUAL, "")); // Add a dummy entry at the end. + ListIterator pointer = diffs.listIterator(); + int count_delete = 0; + int count_insert = 0; + String text_delete = ""; + String text_insert = ""; + Diff thisDiff = pointer.next(); + Diff prevEqual = null; + int commonlength; + while (thisDiff != null) { + switch (thisDiff.operation) { + case INSERT: + count_insert++; + text_insert += thisDiff.text; + prevEqual = null; + break; + case DELETE: + count_delete++; + text_delete += thisDiff.text; + prevEqual = null; + break; + case EQUAL: + if (count_delete + count_insert > 1) { + boolean both_types = count_delete != 0 && count_insert != 0; + // Delete the offending records. + pointer.previous(); // Reverse direction. + while (count_delete-- > 0) { + pointer.previous(); + pointer.remove(); + } + while (count_insert-- > 0) { + pointer.previous(); + pointer.remove(); + } + if (both_types) { + // Factor out any common prefixies. + commonlength = diff_commonPrefix(text_insert, text_delete); + if (commonlength != 0) { + if (pointer.hasPrevious()) { + thisDiff = pointer.previous(); + assert thisDiff.operation == Operation.EQUAL + : "Previous diff should have been an equality."; + thisDiff.text += text_insert.substring(0, commonlength); + pointer.next(); + } else { + pointer.add(new Diff(Operation.EQUAL, + text_insert.substring(0, commonlength))); + } + text_insert = text_insert.substring(commonlength); + text_delete = text_delete.substring(commonlength); + } + // Factor out any common suffixies. + commonlength = diff_commonSuffix(text_insert, text_delete); + if (commonlength != 0) { + thisDiff = pointer.next(); + thisDiff.text = text_insert.substring(text_insert.length() + - commonlength) + thisDiff.text; + text_insert = text_insert.substring(0, text_insert.length() + - commonlength); + text_delete = text_delete.substring(0, text_delete.length() + - commonlength); + pointer.previous(); + } + } + // Insert the merged records. + if (text_delete.length() != 0) { + pointer.add(new Diff(Operation.DELETE, text_delete)); + } + if (text_insert.length() != 0) { + pointer.add(new Diff(Operation.INSERT, text_insert)); + } + // Step forward to the equality. + thisDiff = pointer.hasNext() ? pointer.next() : null; + } else if (prevEqual != null) { + // Merge this equality with the previous one. + prevEqual.text += thisDiff.text; + pointer.remove(); + thisDiff = pointer.previous(); + pointer.next(); // Forward direction + } + count_insert = 0; + count_delete = 0; + text_delete = ""; + text_insert = ""; + prevEqual = thisDiff; + break; + } + thisDiff = pointer.hasNext() ? pointer.next() : null; + } + if (diffs.getLast().text.length() == 0) { + diffs.removeLast(); // Remove the dummy entry at the end. + } + + /* + * Second pass: look for single edits surrounded on both sides by equalities + * which can be shifted sideways to eliminate an equality. + * e.g: ABAC -> ABAC + */ + boolean changes = false; + // Create a new iterator at the start. + // (As opposed to walking the current one back.) + pointer = diffs.listIterator(); + Diff prevDiff = pointer.hasNext() ? pointer.next() : null; + thisDiff = pointer.hasNext() ? pointer.next() : null; + Diff nextDiff = pointer.hasNext() ? pointer.next() : null; + // Intentionally ignore the first and last element (don't need checking). + while (nextDiff != null) { + if (prevDiff.operation == Operation.EQUAL && + nextDiff.operation == Operation.EQUAL) { + // This is a single edit surrounded by equalities. + if (thisDiff.text.endsWith(prevDiff.text)) { + // Shift the edit over the previous equality. + thisDiff.text = prevDiff.text + + thisDiff.text.substring(0, thisDiff.text.length() + - prevDiff.text.length()); + nextDiff.text = prevDiff.text + nextDiff.text; + pointer.previous(); // Walk past nextDiff. + pointer.previous(); // Walk past thisDiff. + pointer.previous(); // Walk past prevDiff. + pointer.remove(); // Delete prevDiff. + pointer.next(); // Walk past thisDiff. + thisDiff = pointer.next(); // Walk past nextDiff. + nextDiff = pointer.hasNext() ? pointer.next() : null; + changes = true; + } else if (thisDiff.text.startsWith(nextDiff.text)) { + // Shift the edit over the next equality. + prevDiff.text += nextDiff.text; + thisDiff.text = thisDiff.text.substring(nextDiff.text.length()) + + nextDiff.text; + pointer.remove(); // Delete nextDiff. + nextDiff = pointer.hasNext() ? pointer.next() : null; + changes = true; + } + } + prevDiff = thisDiff; + thisDiff = nextDiff; + nextDiff = pointer.hasNext() ? pointer.next() : null; + } + // If shifts were made, the diff needs reordering and another shift sweep. + if (changes) { + diff_cleanupMerge(diffs); + } + } + + /** + * loc is a location in text1, compute and return the equivalent location in + * text2. + * e.g. "The cat" vs "The big cat", 1->1, 5->8 + * @param diffs List of Diff objects. + * @param loc Location within text1. + * @return Location within text2. + */ + public int diff_xIndex(List diffs, int loc) { + int chars1 = 0; + int chars2 = 0; + int last_chars1 = 0; + int last_chars2 = 0; + Diff lastDiff = null; + for (Diff aDiff : diffs) { + if (aDiff.operation != Operation.INSERT) { + // Equality or deletion. + chars1 += aDiff.text.length(); + } + if (aDiff.operation != Operation.DELETE) { + // Equality or insertion. + chars2 += aDiff.text.length(); + } + if (chars1 > loc) { + // Overshot the location. + lastDiff = aDiff; + break; + } + last_chars1 = chars1; + last_chars2 = chars2; + } + if (lastDiff != null && lastDiff.operation == Operation.DELETE) { + // The location was deleted. + return last_chars2; + } + // Add the remaining character length. + return last_chars2 + (loc - last_chars1); + } + + /** + * Convert a Diff list into a pretty HTML report. + * @param diffs List of Diff objects. + * @return HTML representation. + */ + public String diff_prettyHtml(List diffs) { + StringBuilder html = new StringBuilder(); + for (Diff aDiff : diffs) { + String text = aDiff.text.replace("&", "&").replace("<", "<") + .replace(">", ">").replace("\n", "¶
"); + switch (aDiff.operation) { + case INSERT: + html.append("").append(text) + .append(""); + break; + case DELETE: + html.append("").append(text) + .append(""); + break; + case EQUAL: + html.append("").append(text).append(""); + break; + } + } + return html.toString(); + } + + /** + * Compute and return the source text (all equalities and deletions). + * @param diffs List of Diff objects. + * @return Source text. + */ + public String diff_text1(List diffs) { + StringBuilder text = new StringBuilder(); + for (Diff aDiff : diffs) { + if (aDiff.operation != Operation.INSERT) { + text.append(aDiff.text); + } + } + return text.toString(); + } + + /** + * Compute and return the destination text (all equalities and insertions). + * @param diffs List of Diff objects. + * @return Destination text. + */ + public String diff_text2(List diffs) { + StringBuilder text = new StringBuilder(); + for (Diff aDiff : diffs) { + if (aDiff.operation != Operation.DELETE) { + text.append(aDiff.text); + } + } + return text.toString(); + } + + /** + * Compute the Levenshtein distance; the number of inserted, deleted or + * substituted characters. + * @param diffs List of Diff objects. + * @return Number of changes. + */ + public int diff_levenshtein(List diffs) { + int levenshtein = 0; + int insertions = 0; + int deletions = 0; + for (Diff aDiff : diffs) { + switch (aDiff.operation) { + case INSERT: + insertions += aDiff.text.length(); + break; + case DELETE: + deletions += aDiff.text.length(); + break; + case EQUAL: + // A deletion and an insertion is one substitution. + levenshtein += Math.max(insertions, deletions); + insertions = 0; + deletions = 0; + break; + } + } + levenshtein += Math.max(insertions, deletions); + return levenshtein; + } + + /** + * Crush the diff into an encoded string which describes the operations + * required to transform text1 into text2. + * E.g. =3\t-2\t+ing -> Keep 3 chars, delete 2 chars, insert 'ing'. + * Operations are tab-separated. Inserted text is escaped using %xx notation. + * @param diffs List of Diff objects. + * @return Delta text. + */ + public String diff_toDelta(List diffs) { + StringBuilder text = new StringBuilder(); + for (Diff aDiff : diffs) { + switch (aDiff.operation) { + case INSERT: + try { + text.append("+").append(URLEncoder.encode(aDiff.text, "UTF-8") + .replace('+', ' ')).append("\t"); + } catch (UnsupportedEncodingException e) { + // Not likely on modern system. + throw new Error("This system does not support UTF-8.", e); + } + break; + case DELETE: + text.append("-").append(aDiff.text.length()).append("\t"); + break; + case EQUAL: + text.append("=").append(aDiff.text.length()).append("\t"); + break; + } + } + String delta = text.toString(); + if (delta.length() != 0) { + // Strip off trailing tab character. + delta = delta.substring(0, delta.length() - 1); + delta = unescapeForEncodeUriCompatability(delta); + } + return delta; + } + + /** + * Given the original text1, and an encoded string which describes the + * operations required to transform text1 into text2, compute the full diff. + * @param text1 Source string for the diff. + * @param delta Delta text. + * @return Array of Diff objects or null if invalid. + * @throws IllegalArgumentException If invalid input. + */ + public LinkedList diff_fromDelta(String text1, String delta) + throws IllegalArgumentException { + LinkedList diffs = new LinkedList(); + int pointer = 0; // Cursor in text1 + String[] tokens = delta.split("\t"); + for (String token : tokens) { + if (token.length() == 0) { + // Blank tokens are ok (from a trailing \t). + continue; + } + // Each token begins with a one character parameter which specifies the + // operation of this token (delete, insert, equality). + String param = token.substring(1); + switch (token.charAt(0)) { + case '+': + // decode would change all "+" to " " + param = param.replace("+", "%2B"); + try { + param = URLDecoder.decode(param, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // Not likely on modern system. + throw new Error("This system does not support UTF-8.", e); + } catch (IllegalArgumentException e) { + // Malformed URI sequence. + throw new IllegalArgumentException( + "Illegal escape in diff_fromDelta: " + param, e); + } + diffs.add(new Diff(Operation.INSERT, param)); + break; + case '-': + // Fall through. + case '=': + int n; + try { + n = Integer.parseInt(param); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Invalid number in diff_fromDelta: " + param, e); + } + if (n < 0) { + throw new IllegalArgumentException( + "Negative number in diff_fromDelta: " + param); + } + String text; + try { + text = text1.substring(pointer, pointer += n); + } catch (StringIndexOutOfBoundsException e) { + throw new IllegalArgumentException("Delta length (" + pointer + + ") larger than source text length (" + text1.length() + + ").", e); + } + if (token.charAt(0) == '=') { + diffs.add(new Diff(Operation.EQUAL, text)); + } else { + diffs.add(new Diff(Operation.DELETE, text)); + } + break; + default: + // Anything else is an error. + throw new IllegalArgumentException( + "Invalid diff operation in diff_fromDelta: " + token.charAt(0)); + } + } + if (pointer != text1.length()) { + throw new IllegalArgumentException("Delta length (" + pointer + + ") smaller than source text length (" + text1.length() + ")."); + } + return diffs; + } + + + // MATCH FUNCTIONS + + + /** + * Locate the best instance of 'pattern' in 'text' near 'loc'. + * Returns -1 if no match found. + * @param text The text to search. + * @param pattern The pattern to search for. + * @param loc The location to search around. + * @return Best match index or -1. + */ + public int match_main(String text, String pattern, int loc) { + // Check for null inputs. + if (text == null || pattern == null) { + throw new IllegalArgumentException("Null inputs. (match_main)"); + } + + loc = Math.max(0, Math.min(loc, text.length())); + if (text.equals(pattern)) { + // Shortcut (potentially not guaranteed by the algorithm) + return 0; + } else if (text.length() == 0) { + // Nothing to match. + return -1; + } else if (loc + pattern.length() <= text.length() + && text.substring(loc, loc + pattern.length()).equals(pattern)) { + // Perfect match at the perfect spot! (Includes case of null pattern) + return loc; + } else { + // Do a fuzzy compare. + return match_bitap(text, pattern, loc); + } + } + + /** + * Locate the best instance of 'pattern' in 'text' near 'loc' using the + * Bitap algorithm. Returns -1 if no match found. + * @param text The text to search. + * @param pattern The pattern to search for. + * @param loc The location to search around. + * @return Best match index or -1. + */ + protected int match_bitap(String text, String pattern, int loc) { + assert (Match_MaxBits == 0 || pattern.length() <= Match_MaxBits) + : "Pattern too long for this application."; + + // Initialise the alphabet. + Map s = match_alphabet(pattern); + + // Highest score beyond which we give up. + double score_threshold = Match_Threshold; + // Is there a nearby exact match? (speedup) + int best_loc = text.indexOf(pattern, loc); + if (best_loc != -1) { + score_threshold = Math.min(match_bitapScore(0, best_loc, loc, pattern), + score_threshold); + // What about in the other direction? (speedup) + best_loc = text.lastIndexOf(pattern, loc + pattern.length()); + if (best_loc != -1) { + score_threshold = Math.min(match_bitapScore(0, best_loc, loc, pattern), + score_threshold); + } + } + + // Initialise the bit arrays. + int matchmask = 1 << (pattern.length() - 1); + best_loc = -1; + + int bin_min, bin_mid; + int bin_max = pattern.length() + text.length(); + // Empty initialization added to appease Java compiler. + int[] last_rd = new int[0]; + for (int d = 0; d < pattern.length(); d++) { + // Scan for the best match; each iteration allows for one more error. + // Run a binary search to determine how far from 'loc' we can stray at + // this error level. + bin_min = 0; + bin_mid = bin_max; + while (bin_min < bin_mid) { + if (match_bitapScore(d, loc + bin_mid, loc, pattern) + <= score_threshold) { + bin_min = bin_mid; + } else { + bin_max = bin_mid; + } + bin_mid = (bin_max - bin_min) / 2 + bin_min; + } + // Use the result from this iteration as the maximum for the next. + bin_max = bin_mid; + int start = Math.max(1, loc - bin_mid + 1); + int finish = Math.min(loc + bin_mid, text.length()) + pattern.length(); + + int[] rd = new int[finish + 2]; + rd[finish + 1] = (1 << d) - 1; + for (int j = finish; j >= start; j--) { + int charMatch; + if (text.length() <= j - 1 || !s.containsKey(text.charAt(j - 1))) { + // Out of range. + charMatch = 0; + } else { + charMatch = s.get(text.charAt(j - 1)); + } + if (d == 0) { + // First pass: exact match. + rd[j] = ((rd[j + 1] << 1) | 1) & charMatch; + } else { + // Subsequent passes: fuzzy match. + rd[j] = (((rd[j + 1] << 1) | 1) & charMatch) + | (((last_rd[j + 1] | last_rd[j]) << 1) | 1) | last_rd[j + 1]; + } + if ((rd[j] & matchmask) != 0) { + double score = match_bitapScore(d, j - 1, loc, pattern); + // This match will almost certainly be better than any existing + // match. But check anyway. + if (score <= score_threshold) { + // Told you so. + score_threshold = score; + best_loc = j - 1; + if (best_loc > loc) { + // When passing loc, don't exceed our current distance from loc. + start = Math.max(1, 2 * loc - best_loc); + } else { + // Already passed loc, downhill from here on in. + break; + } + } + } + } + if (match_bitapScore(d + 1, loc, loc, pattern) > score_threshold) { + // No hope for a (better) match at greater error levels. + break; + } + last_rd = rd; + } + return best_loc; + } + + /** + * Compute and return the score for a match with e errors and x location. + * @param e Number of errors in match. + * @param x Location of match. + * @param loc Expected location of match. + * @param pattern Pattern being sought. + * @return Overall score for match (0.0 = good, 1.0 = bad). + */ + private double match_bitapScore(int e, int x, int loc, String pattern) { + float accuracy = (float) e / pattern.length(); + int proximity = Math.abs(loc - x); + if (Match_Distance == 0) { + // Dodge divide by zero error. + return proximity == 0 ? accuracy : 1.0; + } + return accuracy + (proximity / (float) Match_Distance); + } + + /** + * Initialise the alphabet for the Bitap algorithm. + * @param pattern The text to encode. + * @return Hash of character locations. + */ + protected Map match_alphabet(String pattern) { + Map s = new HashMap(); + char[] char_pattern = pattern.toCharArray(); + for (char c : char_pattern) { + s.put(c, 0); + } + int i = 0; + for (char c : char_pattern) { + s.put(c, s.get(c) | (1 << (pattern.length() - i - 1))); + i++; + } + return s; + } + + + // PATCH FUNCTIONS + + + /** + * Increase the context until it is unique, + * but don't let the pattern expand beyond Match_MaxBits. + * @param patch The patch to grow. + * @param text Source text. + */ + protected void patch_addContext(Patch patch, String text) { + if (text.length() == 0) { + return; + } + String pattern = text.substring(patch.start2, patch.start2 + patch.length1); + int padding = 0; + + // Look for the first and last matches of pattern in text. If two different + // matches are found, increase the pattern length. + while (text.indexOf(pattern) != text.lastIndexOf(pattern) + && pattern.length() < Match_MaxBits - Patch_Margin - Patch_Margin) { + padding += Patch_Margin; + pattern = text.substring(Math.max(0, patch.start2 - padding), + Math.min(text.length(), patch.start2 + patch.length1 + padding)); + } + // Add one chunk for good luck. + padding += Patch_Margin; + + // Add the prefix. + String prefix = text.substring(Math.max(0, patch.start2 - padding), + patch.start2); + if (prefix.length() != 0) { + patch.diffs.addFirst(new Diff(Operation.EQUAL, prefix)); + } + // Add the suffix. + String suffix = text.substring(patch.start2 + patch.length1, + Math.min(text.length(), patch.start2 + patch.length1 + padding)); + if (suffix.length() != 0) { + patch.diffs.addLast(new Diff(Operation.EQUAL, suffix)); + } + + // Roll back the start points. + patch.start1 -= prefix.length(); + patch.start2 -= prefix.length(); + // Extend the lengths. + patch.length1 += prefix.length() + suffix.length(); + patch.length2 += prefix.length() + suffix.length(); + } + + /** + * Compute a list of patches to turn text1 into text2. + * A set of diffs will be computed. + * @param text1 Old text. + * @param text2 New text. + * @return LinkedList of Patch objects. + */ + public LinkedList patch_make(String text1, String text2) { + if (text1 == null || text2 == null) { + throw new IllegalArgumentException("Null inputs. (patch_make)"); + } + // No diffs provided, compute our own. + LinkedList diffs = diff_main(text1, text2, true); + if (diffs.size() > 2) { + diff_cleanupSemantic(diffs); + diff_cleanupEfficiency(diffs); + } + return patch_make(text1, diffs); + } + + /** + * Compute a list of patches to turn text1 into text2. + * text1 will be derived from the provided diffs. + * @param diffs Array of Diff objects for text1 to text2. + * @return LinkedList of Patch objects. + */ + public LinkedList patch_make(LinkedList diffs) { + if (diffs == null) { + throw new IllegalArgumentException("Null inputs. (patch_make)"); + } + // No origin string provided, compute our own. + String text1 = diff_text1(diffs); + return patch_make(text1, diffs); + } + + /** + * Compute a list of patches to turn text1 into text2. + * text2 is ignored, diffs are the delta between text1 and text2. + * @param text1 Old text + * @param text2 Ignored. + * @param diffs Array of Diff objects for text1 to text2. + * @return LinkedList of Patch objects. + * @deprecated Prefer patch_make(String text1, LinkedList diffs). + */ + @Deprecated public LinkedList patch_make(String text1, String text2, + LinkedList diffs) { + return patch_make(text1, diffs); + } + + /** + * Compute a list of patches to turn text1 into text2. + * text2 is not provided, diffs are the delta between text1 and text2. + * @param text1 Old text. + * @param diffs Array of Diff objects for text1 to text2. + * @return LinkedList of Patch objects. + */ + public LinkedList patch_make(String text1, LinkedList diffs) { + if (text1 == null || diffs == null) { + throw new IllegalArgumentException("Null inputs. (patch_make)"); + } + + LinkedList patches = new LinkedList(); + if (diffs.isEmpty()) { + return patches; // Get rid of the null case. + } + Patch patch = new Patch(); + int char_count1 = 0; // Number of characters into the text1 string. + int char_count2 = 0; // Number of characters into the text2 string. + // Start with text1 (prepatch_text) and apply the diffs until we arrive at + // text2 (postpatch_text). We recreate the patches one by one to determine + // context info. + String prepatch_text = text1; + String postpatch_text = text1; + for (Diff aDiff : diffs) { + if (patch.diffs.isEmpty() && aDiff.operation != Operation.EQUAL) { + // A new patch starts here. + patch.start1 = char_count1; + patch.start2 = char_count2; + } + + switch (aDiff.operation) { + case INSERT: + patch.diffs.add(aDiff); + patch.length2 += aDiff.text.length(); + postpatch_text = postpatch_text.substring(0, char_count2) + + aDiff.text + postpatch_text.substring(char_count2); + break; + case DELETE: + patch.length1 += aDiff.text.length(); + patch.diffs.add(aDiff); + postpatch_text = postpatch_text.substring(0, char_count2) + + postpatch_text.substring(char_count2 + aDiff.text.length()); + break; + case EQUAL: + if (aDiff.text.length() <= 2 * Patch_Margin + && !patch.diffs.isEmpty() && aDiff != diffs.getLast()) { + // Small equality inside a patch. + patch.diffs.add(aDiff); + patch.length1 += aDiff.text.length(); + patch.length2 += aDiff.text.length(); + } + + if (aDiff.text.length() >= 2 * Patch_Margin && !patch.diffs.isEmpty()) { + // Time for a new patch. + if (!patch.diffs.isEmpty()) { + patch_addContext(patch, prepatch_text); + patches.add(patch); + patch = new Patch(); + // Unlike Unidiff, our patch lists have a rolling context. + // https://github.com/google/diff-match-patch/wiki/Unidiff + // Update prepatch text & pos to reflect the application of the + // just completed patch. + prepatch_text = postpatch_text; + char_count1 = char_count2; + } + } + break; + } + + // Update the current character count. + if (aDiff.operation != Operation.INSERT) { + char_count1 += aDiff.text.length(); + } + if (aDiff.operation != Operation.DELETE) { + char_count2 += aDiff.text.length(); + } + } + // Pick up the leftover patch if not empty. + if (!patch.diffs.isEmpty()) { + patch_addContext(patch, prepatch_text); + patches.add(patch); + } + + return patches; + } + + /** + * Given an array of patches, return another array that is identical. + * @param patches Array of Patch objects. + * @return Array of Patch objects. + */ + public LinkedList patch_deepCopy(LinkedList patches) { + LinkedList patchesCopy = new LinkedList(); + for (Patch aPatch : patches) { + Patch patchCopy = new Patch(); + for (Diff aDiff : aPatch.diffs) { + Diff diffCopy = new Diff(aDiff.operation, aDiff.text); + patchCopy.diffs.add(diffCopy); + } + patchCopy.start1 = aPatch.start1; + patchCopy.start2 = aPatch.start2; + patchCopy.length1 = aPatch.length1; + patchCopy.length2 = aPatch.length2; + patchesCopy.add(patchCopy); + } + return patchesCopy; + } + + /** + * Merge a set of patches onto the text. Return a patched text, as well + * as an array of true/false values indicating which patches were applied. + * @param patches Array of Patch objects + * @param text Old text. + * @return Two element Object array, containing the new text and an array of + * boolean values. + */ + public Object[] patch_apply(LinkedList patches, String text) { + if (patches.isEmpty()) { + return new Object[]{text, new boolean[0]}; + } + + // Deep copy the patches so that no changes are made to originals. + patches = patch_deepCopy(patches); + + String nullPadding = patch_addPadding(patches); + text = nullPadding + text + nullPadding; + patch_splitMax(patches); + + int x = 0; + // delta keeps track of the offset between the expected and actual location + // of the previous patch. If there are patches expected at positions 10 and + // 20, but the first patch was found at 12, delta is 2 and the second patch + // has an effective expected position of 22. + int delta = 0; + boolean[] results = new boolean[patches.size()]; + for (Patch aPatch : patches) { + int expected_loc = aPatch.start2 + delta; + String text1 = diff_text1(aPatch.diffs); + int start_loc; + int end_loc = -1; + if (text1.length() > this.Match_MaxBits) { + // patch_splitMax will only provide an oversized pattern in the case of + // a monster delete. + start_loc = match_main(text, + text1.substring(0, this.Match_MaxBits), expected_loc); + if (start_loc != -1) { + end_loc = match_main(text, + text1.substring(text1.length() - this.Match_MaxBits), + expected_loc + text1.length() - this.Match_MaxBits); + if (end_loc == -1 || start_loc >= end_loc) { + // Can't find valid trailing context. Drop this patch. + start_loc = -1; + } + } + } else { + start_loc = match_main(text, text1, expected_loc); + } + if (start_loc == -1) { + // No match found. :( + results[x] = false; + // Subtract the delta for this failed patch from subsequent patches. + delta -= aPatch.length2 - aPatch.length1; + } else { + // Found a match. :) + results[x] = true; + delta = start_loc - expected_loc; + String text2; + if (end_loc == -1) { + text2 = text.substring(start_loc, + Math.min(start_loc + text1.length(), text.length())); + } else { + text2 = text.substring(start_loc, + Math.min(end_loc + this.Match_MaxBits, text.length())); + } + if (text1.equals(text2)) { + // Perfect match, just shove the replacement text in. + text = text.substring(0, start_loc) + diff_text2(aPatch.diffs) + + text.substring(start_loc + text1.length()); + } else { + // Imperfect match. Run a diff to get a framework of equivalent + // indices. + LinkedList diffs = diff_main(text1, text2, false); + if (text1.length() > this.Match_MaxBits + && diff_levenshtein(diffs) / (float) text1.length() + > this.Patch_DeleteThreshold) { + // The end points match, but the content is unacceptably bad. + results[x] = false; + } else { + diff_cleanupSemanticLossless(diffs); + int index1 = 0; + for (Diff aDiff : aPatch.diffs) { + if (aDiff.operation != Operation.EQUAL) { + int index2 = diff_xIndex(diffs, index1); + if (aDiff.operation == Operation.INSERT) { + // Insertion + text = text.substring(0, start_loc + index2) + aDiff.text + + text.substring(start_loc + index2); + } else if (aDiff.operation == Operation.DELETE) { + // Deletion + text = text.substring(0, start_loc + index2) + + text.substring(start_loc + diff_xIndex(diffs, + index1 + aDiff.text.length())); + } + } + if (aDiff.operation != Operation.DELETE) { + index1 += aDiff.text.length(); + } + } + } + } + } + x++; + } + // Strip the padding off. + text = text.substring(nullPadding.length(), text.length() + - nullPadding.length()); + return new Object[]{text, results}; + } + + /** + * Add some padding on text start and end so that edges can match something. + * Intended to be called only from within patch_apply. + * @param patches Array of Patch objects. + * @return The padding string added to each side. + */ + public String patch_addPadding(LinkedList patches) { + short paddingLength = this.Patch_Margin; + String nullPadding = ""; + for (short x = 1; x <= paddingLength; x++) { + nullPadding += String.valueOf((char) x); + } + + // Bump all the patches forward. + for (Patch aPatch : patches) { + aPatch.start1 += paddingLength; + aPatch.start2 += paddingLength; + } + + // Add some padding on start of first diff. + Patch patch = patches.getFirst(); + LinkedList diffs = patch.diffs; + if (diffs.isEmpty() || diffs.getFirst().operation != Operation.EQUAL) { + // Add nullPadding equality. + diffs.addFirst(new Diff(Operation.EQUAL, nullPadding)); + patch.start1 -= paddingLength; // Should be 0. + patch.start2 -= paddingLength; // Should be 0. + patch.length1 += paddingLength; + patch.length2 += paddingLength; + } else if (paddingLength > diffs.getFirst().text.length()) { + // Grow first equality. + Diff firstDiff = diffs.getFirst(); + int extraLength = paddingLength - firstDiff.text.length(); + firstDiff.text = nullPadding.substring(firstDiff.text.length()) + + firstDiff.text; + patch.start1 -= extraLength; + patch.start2 -= extraLength; + patch.length1 += extraLength; + patch.length2 += extraLength; + } + + // Add some padding on end of last diff. + patch = patches.getLast(); + diffs = patch.diffs; + if (diffs.isEmpty() || diffs.getLast().operation != Operation.EQUAL) { + // Add nullPadding equality. + diffs.addLast(new Diff(Operation.EQUAL, nullPadding)); + patch.length1 += paddingLength; + patch.length2 += paddingLength; + } else if (paddingLength > diffs.getLast().text.length()) { + // Grow last equality. + Diff lastDiff = diffs.getLast(); + int extraLength = paddingLength - lastDiff.text.length(); + lastDiff.text += nullPadding.substring(0, extraLength); + patch.length1 += extraLength; + patch.length2 += extraLength; + } + + return nullPadding; + } + + /** + * Look through the patches and break up any which are longer than the + * maximum limit of the match algorithm. + * Intended to be called only from within patch_apply. + * @param patches LinkedList of Patch objects. + */ + public void patch_splitMax(LinkedList patches) { + short patch_size = Match_MaxBits; + String precontext, postcontext; + Patch patch; + int start1, start2; + boolean empty; + Operation diff_type; + String diff_text; + ListIterator pointer = patches.listIterator(); + Patch bigpatch = pointer.hasNext() ? pointer.next() : null; + while (bigpatch != null) { + if (bigpatch.length1 <= Match_MaxBits) { + bigpatch = pointer.hasNext() ? pointer.next() : null; + continue; + } + // Remove the big old patch. + pointer.remove(); + start1 = bigpatch.start1; + start2 = bigpatch.start2; + precontext = ""; + while (!bigpatch.diffs.isEmpty()) { + // Create one of several smaller patches. + patch = new Patch(); + empty = true; + patch.start1 = start1 - precontext.length(); + patch.start2 = start2 - precontext.length(); + if (precontext.length() != 0) { + patch.length1 = patch.length2 = precontext.length(); + patch.diffs.add(new Diff(Operation.EQUAL, precontext)); + } + while (!bigpatch.diffs.isEmpty() + && patch.length1 < patch_size - Patch_Margin) { + diff_type = bigpatch.diffs.getFirst().operation; + diff_text = bigpatch.diffs.getFirst().text; + if (diff_type == Operation.INSERT) { + // Insertions are harmless. + patch.length2 += diff_text.length(); + start2 += diff_text.length(); + patch.diffs.addLast(bigpatch.diffs.removeFirst()); + empty = false; + } else if (diff_type == Operation.DELETE && patch.diffs.size() == 1 + && patch.diffs.getFirst().operation == Operation.EQUAL + && diff_text.length() > 2 * patch_size) { + // This is a large deletion. Let it pass in one chunk. + patch.length1 += diff_text.length(); + start1 += diff_text.length(); + empty = false; + patch.diffs.add(new Diff(diff_type, diff_text)); + bigpatch.diffs.removeFirst(); + } else { + // Deletion or equality. Only take as much as we can stomach. + diff_text = diff_text.substring(0, Math.min(diff_text.length(), + patch_size - patch.length1 - Patch_Margin)); + patch.length1 += diff_text.length(); + start1 += diff_text.length(); + if (diff_type == Operation.EQUAL) { + patch.length2 += diff_text.length(); + start2 += diff_text.length(); + } else { + empty = false; + } + patch.diffs.add(new Diff(diff_type, diff_text)); + if (diff_text.equals(bigpatch.diffs.getFirst().text)) { + bigpatch.diffs.removeFirst(); + } else { + bigpatch.diffs.getFirst().text = bigpatch.diffs.getFirst().text + .substring(diff_text.length()); + } + } + } + // Compute the head context for the next patch. + precontext = diff_text2(patch.diffs); + precontext = precontext.substring(Math.max(0, precontext.length() + - Patch_Margin)); + // Append the end context for this patch. + if (diff_text1(bigpatch.diffs).length() > Patch_Margin) { + postcontext = diff_text1(bigpatch.diffs).substring(0, Patch_Margin); + } else { + postcontext = diff_text1(bigpatch.diffs); + } + if (postcontext.length() != 0) { + patch.length1 += postcontext.length(); + patch.length2 += postcontext.length(); + if (!patch.diffs.isEmpty() + && patch.diffs.getLast().operation == Operation.EQUAL) { + patch.diffs.getLast().text += postcontext; + } else { + patch.diffs.add(new Diff(Operation.EQUAL, postcontext)); + } + } + if (!empty) { + pointer.add(patch); + } + } + bigpatch = pointer.hasNext() ? pointer.next() : null; + } + } + + /** + * Take a list of patches and return a textual representation. + * @param patches List of Patch objects. + * @return Text representation of patches. + */ + public String patch_toText(List patches) { + StringBuilder text = new StringBuilder(); + for (Patch aPatch : patches) { + text.append(aPatch); + } + return text.toString(); + } + + /** + * Parse a textual representation of patches and return a List of Patch + * objects. + * @param textline Text representation of patches. + * @return List of Patch objects. + * @throws IllegalArgumentException If invalid input. + */ + public List patch_fromText(String textline) + throws IllegalArgumentException { + List patches = new LinkedList(); + if (textline.length() == 0) { + return patches; + } + List textList = Arrays.asList(textline.split("\n")); + LinkedList text = new LinkedList(textList); + Patch patch; + Pattern patchHeader + = Pattern.compile("^@@ -(\\d+),?(\\d*) \\+(\\d+),?(\\d*) @@$"); + Matcher m; + char sign; + String line; + while (!text.isEmpty()) { + m = patchHeader.matcher(text.getFirst()); + if (!m.matches()) { + throw new IllegalArgumentException( + "Invalid patch string: " + text.getFirst()); + } + patch = new Patch(); + patches.add(patch); + patch.start1 = Integer.parseInt(m.group(1)); + if (m.group(2).length() == 0) { + patch.start1--; + patch.length1 = 1; + } else if (m.group(2).equals("0")) { + patch.length1 = 0; + } else { + patch.start1--; + patch.length1 = Integer.parseInt(m.group(2)); + } + + patch.start2 = Integer.parseInt(m.group(3)); + if (m.group(4).length() == 0) { + patch.start2--; + patch.length2 = 1; + } else if (m.group(4).equals("0")) { + patch.length2 = 0; + } else { + patch.start2--; + patch.length2 = Integer.parseInt(m.group(4)); + } + text.removeFirst(); + + while (!text.isEmpty()) { + try { + sign = text.getFirst().charAt(0); + } catch (IndexOutOfBoundsException e) { + // Blank line? Whatever. + text.removeFirst(); + continue; + } + line = text.getFirst().substring(1); + line = line.replace("+", "%2B"); // decode would change all "+" to " " + try { + line = URLDecoder.decode(line, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // Not likely on modern system. + throw new Error("This system does not support UTF-8.", e); + } catch (IllegalArgumentException e) { + // Malformed URI sequence. + throw new IllegalArgumentException( + "Illegal escape in patch_fromText: " + line, e); + } + if (sign == '-') { + // Deletion. + patch.diffs.add(new Diff(Operation.DELETE, line)); + } else if (sign == '+') { + // Insertion. + patch.diffs.add(new Diff(Operation.INSERT, line)); + } else if (sign == ' ') { + // Minor equality. + patch.diffs.add(new Diff(Operation.EQUAL, line)); + } else if (sign == '@') { + // Start of next patch. + break; + } else { + // WTF? + throw new IllegalArgumentException( + "Invalid patch mode '" + sign + "' in: " + line); + } + text.removeFirst(); + } + } + return patches; + } + + + /** + * Class representing one diff operation. + */ + public static class Diff { + /** + * One of: INSERT, DELETE or EQUAL. + */ + public Operation operation; + /** + * The text associated with this diff operation. + */ + public String text; + + /** + * Constructor. Initializes the diff with the provided values. + * @param operation One of INSERT, DELETE or EQUAL. + * @param text The text being applied. + */ + public Diff(Operation operation, String text) { + // Construct a diff with the specified operation and text. + this.operation = operation; + this.text = text; + } + + /** + * Display a human-readable version of this Diff. + * @return text version. + */ + public String toString() { + String prettyText = this.text.replace('\n', '\u00b6'); + return "Diff(" + this.operation + ",\"" + prettyText + "\")"; + } + + /** + * Create a numeric hash value for a Diff. + * This function is not used by DMP. + * @return Hash value. + */ + @Override + public int hashCode() { + final int prime = 31; + int result = (operation == null) ? 0 : operation.hashCode(); + result += prime * ((text == null) ? 0 : text.hashCode()); + return result; + } + + /** + * Is this Diff equivalent to another Diff? + * @param obj Another Diff to compare against. + * @return true or false. + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Diff other = (Diff) obj; + if (operation != other.operation) { + return false; + } + if (text == null) { + if (other.text != null) { + return false; + } + } else if (!text.equals(other.text)) { + return false; + } + return true; + } + } + + + /** + * Class representing one patch operation. + */ + public static class Patch { + public LinkedList diffs; + public int start1; + public int start2; + public int length1; + public int length2; + + /** + * Constructor. Initializes with an empty list of diffs. + */ + public Patch() { + this.diffs = new LinkedList(); + } + + /** + * Emulate GNU diff's format. + * Header: @@ -382,8 +481,9 @@ + * Indices are printed as 1-based, not 0-based. + * @return The GNU diff string. + */ + public String toString() { + String coords1, coords2; + if (this.length1 == 0) { + coords1 = this.start1 + ",0"; + } else if (this.length1 == 1) { + coords1 = Integer.toString(this.start1 + 1); + } else { + coords1 = (this.start1 + 1) + "," + this.length1; + } + if (this.length2 == 0) { + coords2 = this.start2 + ",0"; + } else if (this.length2 == 1) { + coords2 = Integer.toString(this.start2 + 1); + } else { + coords2 = (this.start2 + 1) + "," + this.length2; + } + StringBuilder text = new StringBuilder(); + text.append("@@ -").append(coords1).append(" +").append(coords2) + .append(" @@\n"); + // Escape the body of the patch with %xx notation. + for (Diff aDiff : this.diffs) { + switch (aDiff.operation) { + case INSERT: + text.append('+'); + break; + case DELETE: + text.append('-'); + break; + case EQUAL: + text.append(' '); + break; + } + try { + text.append(URLEncoder.encode(aDiff.text, "UTF-8").replace('+', ' ')) + .append("\n"); + } catch (UnsupportedEncodingException e) { + // Not likely on modern system. + throw new Error("This system does not support UTF-8.", e); + } + } + return unescapeForEncodeUriCompatability(text.toString()); + } + } + + /** + * Unescape selected chars for compatability with JavaScript's encodeURI. + * In speed critical applications this could be dropped since the + * receiving application will certainly decode these fine. + * Note that this function is case-sensitive. Thus "%3f" would not be + * unescaped. But this is ok because it is only called with the output of + * URLEncoder.encode which returns uppercase hex. + * + * Example: "%3F" -> "?", "%24" -> "$", etc. + * + * @param str The string to escape. + * @return The escaped string. + */ + private static String unescapeForEncodeUriCompatability(String str) { + return str.replace("%21", "!").replace("%7E", "~") + .replace("%27", "'").replace("%28", "(").replace("%29", ")") + .replace("%3B", ";").replace("%2F", "/").replace("%3F", "?") + .replace("%3A", ":").replace("%40", "@").replace("%26", "&") + .replace("%3D", "=").replace("%2B", "+").replace("%24", "$") + .replace("%2C", ",").replace("%23", "#"); + } +}