forked from D-Net/dnet-hadoop
addition of a sparktester test, implementation of 2 different classes for testing in dnet-dedup-test module, addition of new terms in the vocabulary and change in the implementation of the JaroWinklerNormalizedName comparator
This commit is contained in:
miconis
parent
4bd5a9beee
commit
7e7018c51f
|
|
@ -0,0 +1,252 @@
|
||||||
|
[INFO] Scanning for projects...
|
||||||
|
[INFO] ------------------------------------------------------------------------
|
||||||
|
[INFO] Reactor Build Order:
|
||||||
|
[INFO]
|
||||||
|
[INFO] dnet-dedup [pom]
|
||||||
|
[INFO] dnet-pace-core [jar]
|
||||||
|
[INFO] dnet-dedup-test [jar]
|
||||||
|
[INFO]
|
||||||
|
[INFO] -----------------------< eu.dnetlib:dnet-dedup >------------------------
|
||||||
|
[INFO] Building dnet-dedup 3.0.3-SNAPSHOT [1/3]
|
||||||
|
[INFO] --------------------------------[ pom ]---------------------------------
|
||||||
|
[INFO]
|
||||||
|
[INFO] --- maven-dependency-plugin:3.0.0:tree (default-cli) @ dnet-dedup ---
|
||||||
|
[INFO] eu.dnetlib:dnet-dedup:pom:3.0.3-SNAPSHOT
|
||||||
|
[INFO]
|
||||||
|
[INFO] ---------------------< eu.dnetlib:dnet-pace-core >----------------------
|
||||||
|
[INFO] Building dnet-pace-core 3.0.3-SNAPSHOT [2/3]
|
||||||
|
[INFO] --------------------------------[ jar ]---------------------------------
|
||||||
|
[INFO]
|
||||||
|
[INFO] --- maven-dependency-plugin:3.0.0:tree (default-cli) @ dnet-pace-core ---
|
||||||
|
[INFO] eu.dnetlib:dnet-pace-core:jar:3.0.3-SNAPSHOT
|
||||||
|
[INFO] +- edu.cmu:secondstring:jar:1.0.0:compile
|
||||||
|
[INFO] +- com.google.guava:guava:jar:15.0:compile
|
||||||
|
[INFO] +- com.google.code.gson:gson:jar:2.2.2:compile
|
||||||
|
[INFO] +- commons-lang:commons-lang:jar:2.6:compile
|
||||||
|
[INFO] +- commons-io:commons-io:jar:2.4:compile
|
||||||
|
[INFO] +- commons-collections:commons-collections:jar:3.2.1:compile
|
||||||
|
[INFO] +- com.googlecode.protobuf-java-format:protobuf-java-format:jar:1.2:compile
|
||||||
|
[INFO] +- org.antlr:stringtemplate:jar:3.2:compile
|
||||||
|
[INFO] | \- org.antlr:antlr:jar:2.7.7:compile
|
||||||
|
[INFO] +- commons-logging:commons-logging:jar:1.1.3:compile
|
||||||
|
[INFO] +- junit:junit:jar:4.9:test
|
||||||
|
[INFO] | \- org.hamcrest:hamcrest-core:jar:1.1:test
|
||||||
|
[INFO] +- org.reflections:reflections:jar:0.9.10:compile
|
||||||
|
[INFO] | +- org.javassist:javassist:jar:3.19.0-GA:compile
|
||||||
|
[INFO] | \- com.google.code.findbugs:annotations:jar:2.0.1:compile
|
||||||
|
[INFO] +- com.fasterxml.jackson.core:jackson-databind:jar:2.6.6:compile
|
||||||
|
[INFO] | +- com.fasterxml.jackson.core:jackson-annotations:jar:2.6.0:compile
|
||||||
|
[INFO] | \- com.fasterxml.jackson.core:jackson-core:jar:2.6.6:compile
|
||||||
|
[INFO] +- org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13:compile
|
||||||
|
[INFO] | \- org.codehaus.jackson:jackson-core-asl:jar:1.9.13:compile
|
||||||
|
[INFO] \- org.apache.commons:commons-math3:jar:3.6.1:compile
|
||||||
|
[INFO]
|
||||||
|
[INFO] ---------------------< eu.dnetlib:dnet-dedup-test >---------------------
|
||||||
|
[INFO] Building dnet-dedup-test 3.0.3-SNAPSHOT [3/3]
|
||||||
|
[INFO] --------------------------------[ jar ]---------------------------------
|
||||||
|
[INFO]
|
||||||
|
[INFO] --- maven-dependency-plugin:3.0.0:tree (default-cli) @ dnet-dedup-test ---
|
||||||
|
[INFO] eu.dnetlib:dnet-dedup-test:jar:3.0.3-SNAPSHOT
|
||||||
|
[INFO] +- eu.dnetlib:dnet-pace-core:jar:3.0.3-SNAPSHOT:compile
|
||||||
|
[INFO] | +- edu.cmu:secondstring:jar:1.0.0:compile
|
||||||
|
[INFO] | +- com.google.guava:guava:jar:15.0:compile
|
||||||
|
[INFO] | +- com.google.code.gson:gson:jar:2.2.2:compile
|
||||||
|
[INFO] | +- commons-lang:commons-lang:jar:2.6:compile
|
||||||
|
[INFO] | +- commons-io:commons-io:jar:2.4:compile
|
||||||
|
[INFO] | +- commons-collections:commons-collections:jar:3.2.1:compile
|
||||||
|
[INFO] | +- com.googlecode.protobuf-java-format:protobuf-java-format:jar:1.2:compile
|
||||||
|
[INFO] | +- org.antlr:stringtemplate:jar:3.2:compile
|
||||||
|
[INFO] | | \- org.antlr:antlr:jar:2.7.7:compile
|
||||||
|
[INFO] | +- commons-logging:commons-logging:jar:1.1.3:compile
|
||||||
|
[INFO] | +- org.reflections:reflections:jar:0.9.10:compile
|
||||||
|
[INFO] | | +- org.javassist:javassist:jar:3.19.0-GA:compile
|
||||||
|
[INFO] | | \- com.google.code.findbugs:annotations:jar:2.0.1:compile
|
||||||
|
[INFO] | +- com.fasterxml.jackson.core:jackson-databind:jar:2.6.6:compile
|
||||||
|
[INFO] | | +- com.fasterxml.jackson.core:jackson-annotations:jar:2.6.0:compile
|
||||||
|
[INFO] | | \- com.fasterxml.jackson.core:jackson-core:jar:2.6.6:compile
|
||||||
|
[INFO] | +- org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13:compile
|
||||||
|
[INFO] | | \- org.codehaus.jackson:jackson-core-asl:jar:1.9.13:compile
|
||||||
|
[INFO] | \- org.apache.commons:commons-math3:jar:3.6.1:compile
|
||||||
|
[INFO] +- eu.dnetlib:dnet-openaire-data-protos:jar:3.9.3-proto250:compile
|
||||||
|
[INFO] | +- com.google.protobuf:protobuf-java:jar:2.5.0:compile
|
||||||
|
[INFO] | \- log4j:log4j:jar:1.2.17:compile (version selected from constraint [1.2.17,1.2.17])
|
||||||
|
[INFO] +- org.apache.spark:spark-core_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | +- org.apache.avro:avro:jar:1.7.7:provided
|
||||||
|
[INFO] | | +- com.thoughtworks.paranamer:paranamer:jar:2.3:provided
|
||||||
|
[INFO] | | \- org.apache.commons:commons-compress:jar:1.4.1:provided
|
||||||
|
[INFO] | | \- org.tukaani:xz:jar:1.0:provided
|
||||||
|
[INFO] | +- org.apache.avro:avro-mapred:jar:hadoop2:1.7.7:provided
|
||||||
|
[INFO] | | +- org.apache.avro:avro-ipc:jar:1.7.7:provided
|
||||||
|
[INFO] | | \- org.apache.avro:avro-ipc:jar:tests:1.7.7:provided
|
||||||
|
[INFO] | +- com.twitter:chill_2.11:jar:0.8.0:provided
|
||||||
|
[INFO] | | \- com.esotericsoftware:kryo-shaded:jar:3.0.3:provided
|
||||||
|
[INFO] | | +- com.esotericsoftware:minlog:jar:1.3.0:provided
|
||||||
|
[INFO] | | \- org.objenesis:objenesis:jar:2.1:provided
|
||||||
|
[INFO] | +- com.twitter:chill-java:jar:0.8.0:provided
|
||||||
|
[INFO] | +- org.apache.xbean:xbean-asm5-shaded:jar:4.4:provided
|
||||||
|
[INFO] | +- org.apache.hadoop:hadoop-client:jar:2.6.5:provided
|
||||||
|
[INFO] | | +- org.apache.hadoop:hadoop-common:jar:2.6.5:provided
|
||||||
|
[INFO] | | | +- commons-cli:commons-cli:jar:1.2:provided
|
||||||
|
[INFO] | | | +- xmlenc:xmlenc:jar:0.52:provided
|
||||||
|
[INFO] | | | +- commons-httpclient:commons-httpclient:jar:3.1:provided
|
||||||
|
[INFO] | | | +- commons-configuration:commons-configuration:jar:1.6:provided
|
||||||
|
[INFO] | | | | +- commons-digester:commons-digester:jar:1.8:provided
|
||||||
|
[INFO] | | | | | \- commons-beanutils:commons-beanutils:jar:1.7.0:provided
|
||||||
|
[INFO] | | | | \- commons-beanutils:commons-beanutils-core:jar:1.8.0:provided
|
||||||
|
[INFO] | | | +- org.apache.hadoop:hadoop-auth:jar:2.6.5:provided
|
||||||
|
[INFO] | | | | \- org.apache.directory.server:apacheds-kerberos-codec:jar:2.0.0-M15:provided
|
||||||
|
[INFO] | | | | +- org.apache.directory.server:apacheds-i18n:jar:2.0.0-M15:provided
|
||||||
|
[INFO] | | | | +- org.apache.directory.api:api-asn1-api:jar:1.0.0-M20:provided
|
||||||
|
[INFO] | | | | \- org.apache.directory.api:api-util:jar:1.0.0-M20:provided
|
||||||
|
[INFO] | | | +- org.apache.curator:curator-client:jar:2.6.0:provided
|
||||||
|
[INFO] | | | \- org.htrace:htrace-core:jar:3.0.4:provided
|
||||||
|
[INFO] | | +- org.apache.hadoop:hadoop-hdfs:jar:2.6.5:provided
|
||||||
|
[INFO] | | | +- org.mortbay.jetty:jetty-util:jar:6.1.26:provided
|
||||||
|
[INFO] | | | \- xerces:xercesImpl:jar:2.9.1:provided
|
||||||
|
[INFO] | | | \- xml-apis:xml-apis:jar:1.3.04:provided
|
||||||
|
[INFO] | | +- org.apache.hadoop:hadoop-mapreduce-client-app:jar:2.6.5:provided
|
||||||
|
[INFO] | | | +- org.apache.hadoop:hadoop-mapreduce-client-common:jar:2.6.5:provided
|
||||||
|
[INFO] | | | | +- org.apache.hadoop:hadoop-yarn-client:jar:2.6.5:provided
|
||||||
|
[INFO] | | | | \- org.apache.hadoop:hadoop-yarn-server-common:jar:2.6.5:provided
|
||||||
|
[INFO] | | | \- org.apache.hadoop:hadoop-mapreduce-client-shuffle:jar:2.6.5:provided
|
||||||
|
[INFO] | | +- org.apache.hadoop:hadoop-yarn-api:jar:2.6.5:provided
|
||||||
|
[INFO] | | +- org.apache.hadoop:hadoop-mapreduce-client-core:jar:2.6.5:provided
|
||||||
|
[INFO] | | | \- org.apache.hadoop:hadoop-yarn-common:jar:2.6.5:provided
|
||||||
|
[INFO] | | | +- javax.xml.bind:jaxb-api:jar:2.2.2:provided
|
||||||
|
[INFO] | | | | \- javax.xml.stream:stax-api:jar:1.0-2:provided
|
||||||
|
[INFO] | | | +- org.codehaus.jackson:jackson-jaxrs:jar:1.9.13:provided
|
||||||
|
[INFO] | | | \- org.codehaus.jackson:jackson-xc:jar:1.9.13:provided
|
||||||
|
[INFO] | | +- org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:2.6.5:provided
|
||||||
|
[INFO] | | \- org.apache.hadoop:hadoop-annotations:jar:2.6.5:provided
|
||||||
|
[INFO] | +- org.apache.spark:spark-launcher_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | +- org.apache.spark:spark-network-common_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | | \- org.fusesource.leveldbjni:leveldbjni-all:jar:1.8:provided
|
||||||
|
[INFO] | +- org.apache.spark:spark-network-shuffle_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | +- org.apache.spark:spark-unsafe_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | +- net.java.dev.jets3t:jets3t:jar:0.9.3:provided
|
||||||
|
[INFO] | | +- org.apache.httpcomponents:httpcore:jar:4.3.3:provided
|
||||||
|
[INFO] | | +- org.apache.httpcomponents:httpclient:jar:4.3.6:provided
|
||||||
|
[INFO] | | +- javax.activation:activation:jar:1.1.1:provided
|
||||||
|
[INFO] | | +- mx4j:mx4j:jar:3.0.2:provided
|
||||||
|
[INFO] | | +- javax.mail:mail:jar:1.4.7:provided
|
||||||
|
[INFO] | | +- org.bouncycastle:bcprov-jdk15on:jar:1.51:provided
|
||||||
|
[INFO] | | \- com.jamesmurty.utils:java-xmlbuilder:jar:1.0:provided
|
||||||
|
[INFO] | | \- net.iharder:base64:jar:2.3.8:provided
|
||||||
|
[INFO] | +- org.apache.curator:curator-recipes:jar:2.6.0:provided
|
||||||
|
[INFO] | | +- org.apache.curator:curator-framework:jar:2.6.0:provided
|
||||||
|
[INFO] | | \- org.apache.zookeeper:zookeeper:jar:3.4.6:provided
|
||||||
|
[INFO] | +- javax.servlet:javax.servlet-api:jar:3.1.0:provided
|
||||||
|
[INFO] | +- org.apache.commons:commons-lang3:jar:3.5:provided
|
||||||
|
[INFO] | +- com.google.code.findbugs:jsr305:jar:1.3.9:provided
|
||||||
|
[INFO] | +- org.slf4j:slf4j-api:jar:1.7.16:provided
|
||||||
|
[INFO] | +- org.slf4j:jul-to-slf4j:jar:1.7.16:provided
|
||||||
|
[INFO] | +- org.slf4j:jcl-over-slf4j:jar:1.7.16:provided
|
||||||
|
[INFO] | +- org.slf4j:slf4j-log4j12:jar:1.7.16:provided
|
||||||
|
[INFO] | +- com.ning:compress-lzf:jar:1.0.3:provided
|
||||||
|
[INFO] | +- org.xerial.snappy:snappy-java:jar:1.1.2.6:provided
|
||||||
|
[INFO] | +- net.jpountz.lz4:lz4:jar:1.3.0:provided
|
||||||
|
[INFO] | +- org.roaringbitmap:RoaringBitmap:jar:0.5.11:provided
|
||||||
|
[INFO] | +- commons-net:commons-net:jar:2.2:provided
|
||||||
|
[INFO] | +- org.scala-lang:scala-library:jar:2.11.8:provided
|
||||||
|
[INFO] | +- org.json4s:json4s-jackson_2.11:jar:3.2.11:provided
|
||||||
|
[INFO] | | \- org.json4s:json4s-core_2.11:jar:3.2.11:provided
|
||||||
|
[INFO] | | +- org.json4s:json4s-ast_2.11:jar:3.2.11:provided
|
||||||
|
[INFO] | | \- org.scala-lang:scalap:jar:2.11.0:provided
|
||||||
|
[INFO] | | \- org.scala-lang:scala-compiler:jar:2.11.0:provided
|
||||||
|
[INFO] | | +- org.scala-lang.modules:scala-xml_2.11:jar:1.0.1:provided
|
||||||
|
[INFO] | | \- org.scala-lang.modules:scala-parser-combinators_2.11:jar:1.0.1:provided
|
||||||
|
[INFO] | +- org.glassfish.jersey.core:jersey-client:jar:2.22.2:provided
|
||||||
|
[INFO] | | +- javax.ws.rs:javax.ws.rs-api:jar:2.0.1:provided
|
||||||
|
[INFO] | | +- org.glassfish.hk2:hk2-api:jar:2.4.0-b34:provided
|
||||||
|
[INFO] | | | +- org.glassfish.hk2:hk2-utils:jar:2.4.0-b34:provided
|
||||||
|
[INFO] | | | \- org.glassfish.hk2.external:aopalliance-repackaged:jar:2.4.0-b34:provided
|
||||||
|
[INFO] | | +- org.glassfish.hk2.external:javax.inject:jar:2.4.0-b34:provided
|
||||||
|
[INFO] | | \- org.glassfish.hk2:hk2-locator:jar:2.4.0-b34:provided
|
||||||
|
[INFO] | +- org.glassfish.jersey.core:jersey-common:jar:2.22.2:provided
|
||||||
|
[INFO] | | +- javax.annotation:javax.annotation-api:jar:1.2:provided
|
||||||
|
[INFO] | | +- org.glassfish.jersey.bundles.repackaged:jersey-guava:jar:2.22.2:provided
|
||||||
|
[INFO] | | \- org.glassfish.hk2:osgi-resource-locator:jar:1.0.1:provided
|
||||||
|
[INFO] | +- org.glassfish.jersey.core:jersey-server:jar:2.22.2:provided
|
||||||
|
[INFO] | | +- org.glassfish.jersey.media:jersey-media-jaxb:jar:2.22.2:provided
|
||||||
|
[INFO] | | \- javax.validation:validation-api:jar:1.1.0.Final:provided
|
||||||
|
[INFO] | +- org.glassfish.jersey.containers:jersey-container-servlet:jar:2.22.2:provided
|
||||||
|
[INFO] | +- org.glassfish.jersey.containers:jersey-container-servlet-core:jar:2.22.2:provided
|
||||||
|
[INFO] | +- io.netty:netty-all:jar:4.0.43.Final:provided
|
||||||
|
[INFO] | +- io.netty:netty:jar:3.9.9.Final:provided
|
||||||
|
[INFO] | +- com.clearspring.analytics:stream:jar:2.7.0:provided
|
||||||
|
[INFO] | +- io.dropwizard.metrics:metrics-core:jar:3.1.2:provided
|
||||||
|
[INFO] | +- io.dropwizard.metrics:metrics-jvm:jar:3.1.2:provided
|
||||||
|
[INFO] | +- io.dropwizard.metrics:metrics-json:jar:3.1.2:provided
|
||||||
|
[INFO] | +- io.dropwizard.metrics:metrics-graphite:jar:3.1.2:provided
|
||||||
|
[INFO] | +- com.fasterxml.jackson.module:jackson-module-scala_2.11:jar:2.6.5:provided
|
||||||
|
[INFO] | | +- org.scala-lang:scala-reflect:jar:2.11.7:provided
|
||||||
|
[INFO] | | \- com.fasterxml.jackson.module:jackson-module-paranamer:jar:2.6.5:provided
|
||||||
|
[INFO] | +- org.apache.ivy:ivy:jar:2.4.0:provided
|
||||||
|
[INFO] | +- oro:oro:jar:2.0.8:provided
|
||||||
|
[INFO] | +- net.razorvine:pyrolite:jar:4.13:provided
|
||||||
|
[INFO] | +- net.sf.py4j:py4j:jar:0.10.4:provided
|
||||||
|
[INFO] | +- org.apache.spark:spark-tags_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | +- org.apache.commons:commons-crypto:jar:1.0.0:provided
|
||||||
|
[INFO] | \- org.spark-project.spark:unused:jar:1.0.0:provided
|
||||||
|
[INFO] +- org.apache.spark:spark-graphx_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | +- org.apache.spark:spark-mllib-local_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | | \- org.scalanlp:breeze_2.11:jar:0.13.1:provided
|
||||||
|
[INFO] | | +- org.scalanlp:breeze-macros_2.11:jar:0.13.1:provided
|
||||||
|
[INFO] | | +- net.sf.opencsv:opencsv:jar:2.3:provided
|
||||||
|
[INFO] | | +- com.github.rwl:jtransforms:jar:2.4.0:provided
|
||||||
|
[INFO] | | +- org.spire-math:spire_2.11:jar:0.13.0:provided
|
||||||
|
[INFO] | | | +- org.spire-math:spire-macros_2.11:jar:0.13.0:provided
|
||||||
|
[INFO] | | | \- org.typelevel:machinist_2.11:jar:0.6.1:provided
|
||||||
|
[INFO] | | \- com.chuusai:shapeless_2.11:jar:2.3.2:provided
|
||||||
|
[INFO] | | \- org.typelevel:macro-compat_2.11:jar:1.1.1:provided
|
||||||
|
[INFO] | +- com.github.fommil.netlib:core:jar:1.1.2:provided
|
||||||
|
[INFO] | \- net.sourceforge.f2j:arpack_combined_all:jar:0.1:provided
|
||||||
|
[INFO] +- org.apache.spark:spark-sql_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | +- com.univocity:univocity-parsers:jar:2.2.1:provided
|
||||||
|
[INFO] | +- org.apache.spark:spark-sketch_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | +- org.apache.spark:spark-catalyst_2.11:jar:2.2.0:provided
|
||||||
|
[INFO] | | +- org.codehaus.janino:janino:jar:3.0.0:provided
|
||||||
|
[INFO] | | +- org.codehaus.janino:commons-compiler:jar:3.0.0:provided
|
||||||
|
[INFO] | | \- org.antlr:antlr4-runtime:jar:4.5.3:provided
|
||||||
|
[INFO] | +- org.apache.parquet:parquet-column:jar:1.8.2:provided
|
||||||
|
[INFO] | | +- org.apache.parquet:parquet-common:jar:1.8.2:provided
|
||||||
|
[INFO] | | \- org.apache.parquet:parquet-encoding:jar:1.8.2:provided
|
||||||
|
[INFO] | \- org.apache.parquet:parquet-hadoop:jar:1.8.2:provided
|
||||||
|
[INFO] | +- org.apache.parquet:parquet-format:jar:2.3.1:provided
|
||||||
|
[INFO] | \- org.apache.parquet:parquet-jackson:jar:1.8.2:provided
|
||||||
|
[INFO] +- eu.dnetlib:dnet-openaireplus-mapping-utils:jar:6.2.18:test
|
||||||
|
[INFO] | +- com.ximpleware:vtd-xml:jar:2.13.4:test (version selected from constraint [2.12,3.0.0))
|
||||||
|
[INFO] | +- commons-codec:commons-codec:jar:1.9:provided
|
||||||
|
[INFO] | +- dom4j:dom4j:jar:1.6.1:test (version selected from constraint [1.6.1,1.6.1])
|
||||||
|
[INFO] | +- net.sf.supercsv:super-csv:jar:2.4.0:test
|
||||||
|
[INFO] | +- eu.dnetlib:cnr-misc-utils:jar:1.0.6-SNAPSHOT:test (version selected from constraint [1.0.0,2.0.0))
|
||||||
|
[INFO] | | +- jaxen:jaxen:jar:1.1.6:test
|
||||||
|
[INFO] | | +- saxonica:saxon:jar:9.1.0.8:test
|
||||||
|
[INFO] | | +- saxonica:saxon-dom:jar:9.1.0.8:test
|
||||||
|
[INFO] | | +- jgrapht:jgrapht:jar:0.7.2:test
|
||||||
|
[INFO] | | +- net.sf.ehcache:ehcache:jar:2.8.0:test
|
||||||
|
[INFO] | | \- org.springframework:spring-test:jar:4.2.5.RELEASE:test (version selected from constraint [4.2.5.RELEASE,4.2.5.RELEASE])
|
||||||
|
[INFO] | | \- org.springframework:spring-core:jar:4.2.5.RELEASE:test
|
||||||
|
[INFO] | +- eu.dnetlib:dnet-hadoop-commons:jar:2.0.2-SNAPSHOT:test (version selected from constraint [2.0.0,3.0.0))
|
||||||
|
[INFO] | | +- org.apache.hadoop:hadoop-core:jar:2.0.0-mr1-cdh4.7.0:test
|
||||||
|
[INFO] | | | +- commons-el:commons-el:jar:1.0:test
|
||||||
|
[INFO] | | | \- hsqldb:hsqldb:jar:1.8.0.10:test
|
||||||
|
[INFO] | | \- org.springframework:spring-beans:jar:4.2.5.RELEASE:test (version selected from constraint [4.2.5.RELEASE,4.2.5.RELEASE])
|
||||||
|
[INFO] | \- eu.dnetlib:dnet-index-solr-common:jar:1.3.1:test (version selected from constraint [1.0.0,1.3.1])
|
||||||
|
[INFO] | \- org.apache.solr:solr-solrj:jar:4.9.0:test
|
||||||
|
[INFO] | +- org.apache.httpcomponents:httpmime:jar:4.3.1:test
|
||||||
|
[INFO] | \- org.noggit:noggit:jar:0.5:test
|
||||||
|
[INFO] \- junit:junit:jar:4.9:test
|
||||||
|
[INFO] \- org.hamcrest:hamcrest-core:jar:1.1:test
|
||||||
|
[INFO] ------------------------------------------------------------------------
|
||||||
|
[INFO] Reactor Summary:
|
||||||
|
[INFO]
|
||||||
|
[INFO] dnet-dedup 3.0.3-SNAPSHOT .......................... SUCCESS [ 1.152 s]
|
||||||
|
[INFO] dnet-pace-core ..................................... SUCCESS [ 0.117 s]
|
||||||
|
[INFO] dnet-dedup-test 3.0.3-SNAPSHOT ..................... SUCCESS [ 1.407 s]
|
||||||
|
[INFO] ------------------------------------------------------------------------
|
||||||
|
[INFO] BUILD SUCCESS
|
||||||
|
[INFO] ------------------------------------------------------------------------
|
||||||
|
[INFO] Total time: 3.216 s
|
||||||
|
[INFO] Finished at: 2019-03-29T15:02:42+01:00
|
||||||
|
[INFO] ------------------------------------------------------------------------
|
||||||
Binary file not shown.
|
|
@ -10,7 +10,7 @@
|
||||||
<groupId>eu.dnetlib</groupId>
|
<groupId>eu.dnetlib</groupId>
|
||||||
<artifactId>dnet-openaire-data-protos</artifactId>
|
<artifactId>dnet-openaire-data-protos</artifactId>
|
||||||
<packaging>jar</packaging>
|
<packaging>jar</packaging>
|
||||||
<version>3.9.4-CUSTOM</version>
|
<version>3.9.4-proto250</version>
|
||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
<!-- defined also in dnet-parent, here in case we need to override -->
|
<!-- defined also in dnet-parent, here in case we need to override -->
|
||||||
|
|
|
||||||
|
|
@ -64,6 +64,7 @@
|
||||||
<groupId>com.fasterxml.jackson.core</groupId>
|
<groupId>com.fasterxml.jackson.core</groupId>
|
||||||
<artifactId>jackson-databind</artifactId>
|
<artifactId>jackson-databind</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.codehaus.jackson</groupId>
|
<groupId>org.codehaus.jackson</groupId>
|
||||||
<artifactId>jackson-mapper-asl</artifactId>
|
<artifactId>jackson-mapper-asl</artifactId>
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,16 @@
|
||||||
package eu.dnetlib.pace.model;
|
package eu.dnetlib.pace.model;
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.lang.reflect.InvocationTargetException;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import com.google.common.base.Splitter;
|
import com.google.common.base.Splitter;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.Gson;
|
||||||
import eu.dnetlib.pace.config.PaceConfig;
|
import eu.dnetlib.pace.config.PaceConfig;
|
||||||
import eu.dnetlib.pace.config.Type;
|
import eu.dnetlib.pace.config.Type;
|
||||||
import eu.dnetlib.pace.distance.*;
|
import eu.dnetlib.pace.distance.DistanceAlgo;
|
||||||
import eu.dnetlib.pace.distance.algo.*;
|
|
||||||
import eu.dnetlib.pace.util.PaceException;
|
import java.io.Serializable;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The schema is composed by field definitions (FieldDef). Each field has a type, a name, and an associated distance algorithm.
|
* The schema is composed by field definitions (FieldDef). Each field has a type, a name, and an associated distance algorithm.
|
||||||
|
|
@ -60,6 +57,18 @@ public class FieldDef implements Serializable {
|
||||||
return name;
|
return name;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setName(String name) {
|
||||||
|
this.name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPath(String path) {
|
||||||
|
this.path = path;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIgnoreMissing(boolean ignoreMissing) {
|
||||||
|
this.ignoreMissing = ignoreMissing;
|
||||||
|
}
|
||||||
|
|
||||||
public String getPath() {
|
public String getPath() {
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ public class ConfigTest extends AbstractPaceTest {
|
||||||
@Test
|
@Test
|
||||||
public void dedupConfigTest() {
|
public void dedupConfigTest() {
|
||||||
|
|
||||||
DedupConfig load = DedupConfig.load(readFromClasspath("result.pace.conf.json"));
|
DedupConfig load = DedupConfig.load(readFromClasspath("org.curr.conf"));
|
||||||
|
|
||||||
System.out.println(load.toString());
|
System.out.println(load.toString());
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
{
|
||||||
|
"wf" : {
|
||||||
|
"threshold" : "0.9",
|
||||||
|
"dedupRun" : "001",
|
||||||
|
"entityType" : "organization",
|
||||||
|
"orderField" : "legalname",
|
||||||
|
"queueMaxSize" : "2000",
|
||||||
|
"groupMaxSize" : "10",
|
||||||
|
"slidingWindowSize" : "200",
|
||||||
|
"rootBuilder" : [ "organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy" ],
|
||||||
|
"includeChildren" : "true"
|
||||||
|
},
|
||||||
|
"pace" : {
|
||||||
|
"clustering" : [
|
||||||
|
{ "name" : "sortedngrampairs", "fields" : [ "legalname" ], "params" : { "max" : 2, "ngramLen" : "3"} },
|
||||||
|
{ "name" : "suffixprefix", "fields" : [ "legalname" ], "params" : { "max" : 1, "len" : "3" } },
|
||||||
|
{ "name" : "urlclustering", "fields" : [ "websiteurl" ], "params" : { } }
|
||||||
|
],
|
||||||
|
"strictConditions" : [
|
||||||
|
{ "name" : "exactMatch", "fields" : [ "gridid" ] }
|
||||||
|
],
|
||||||
|
"conditions" : [
|
||||||
|
{ "name" : "exactMatch", "fields" : [ "country" ] },
|
||||||
|
{ "name" : "DomainExactMatch", "fields" : [ "websiteurl" ] }
|
||||||
|
],
|
||||||
|
"model" : [
|
||||||
|
{ "name" : "legalname", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value" },
|
||||||
|
{ "name" : "country", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/country/classid" },
|
||||||
|
{ "name" : "legalshortname", "algo" : "JaroWinklerNormalizedName", "type" : "String", "weight" : "0.1", "ignoreMissing" : "true", "path" : "organization/metadata/legalshortname/value" },
|
||||||
|
{ "name" : "legalname", "algo" : "JaroWinklerNormalizedName", "type" : "String", "weight" : "0.9", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value", "params" : {"windowSize" : 4, "threshold" : 0.5} },
|
||||||
|
{ "name" : "websiteurl", "algo" : "Null", "type" : "URL", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/websiteurl/value", "params" : { "host" : 0.5, "path" : 0.5 } },
|
||||||
|
{ "name" : "gridid", "algo" : "Null", "type" : "String", "weight" : "0.0", "ignoreMissing" : "true", "path" : "pid[qualifier#classid = {grid}]/value" }
|
||||||
|
],
|
||||||
|
"blacklists" : { }
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue