From f27655e96c7b5b83440a481883b7633b97188062 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 18 Oct 2018 11:56:26 +0200 Subject: [PATCH] updated maven project structure --- dnet-dedup-test/pom.xml | 109 ++++-------- .../src/main/java/eu/dnetlib/SparkTest.java | 15 +- .../eu/dnetlib/reporter/SparkReporter.java | 8 +- dnet-pace-core/pom.xml | 30 ++-- .../eu/dnetlib/pace/util}/BlockProcessor.java | 2 +- .../java/eu/dnetlib/pace/util}/Reporter.java | 2 +- pom.xml | 168 ++++++++++++++++++ 7 files changed, 221 insertions(+), 113 deletions(-) rename {dnet-dedup-test/src/main/java/eu/dnetlib => dnet-pace-core/src/main/java/eu/dnetlib/pace/util}/BlockProcessor.java (99%) rename {dnet-dedup-test/src/main/java/eu/dnetlib => dnet-pace-core/src/main/java/eu/dnetlib/pace/util}/Reporter.java (87%) create mode 100644 pom.xml diff --git a/dnet-dedup-test/pom.xml b/dnet-dedup-test/pom.xml index b64894a..0fd6308 100644 --- a/dnet-dedup-test/pom.xml +++ b/dnet-dedup-test/pom.xml @@ -1,97 +1,64 @@ - + + 4.0.0 + eu.dnetlib - dnet45-parent - 1.0.0 - + dnet-dedup + 3.0.0-SNAPSHOT + ../pom.xml - eu.dnetlib dnet-dedup-test - 1.0-SNAPSHOT + jar + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + src/main/java/**/*.java + src/main/java/**/*.scala + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - eu.dnetlib - dnet-openaire-data-protos - 3.9.3-proto250 - eu.dnetlib dnet-pace-core - 2.6.8-SNAPSHOT + ${project.version} + + + + eu.dnetlib + dnet-openaire-data-protos org.apache.spark spark-core_2.11 - ${spark.version} org.apache.spark spark-graphx_2.11 - ${spark.version} - - - - - - - - - - - - - - - - - - - - - - - - - 2.2.0 - - - \ No newline at end of file diff --git a/dnet-dedup-test/src/main/java/eu/dnetlib/SparkTest.java b/dnet-dedup-test/src/main/java/eu/dnetlib/SparkTest.java index cda1ddf..ce07444 100644 --- a/dnet-dedup-test/src/main/java/eu/dnetlib/SparkTest.java +++ b/dnet-dedup-test/src/main/java/eu/dnetlib/SparkTest.java @@ -1,11 +1,11 @@ package eu.dnetlib; -import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import eu.dnetlib.graph.GraphProcessor; import eu.dnetlib.pace.clustering.BlacklistAwareClusteringCombiner; import eu.dnetlib.pace.config.DedupConfig; import eu.dnetlib.pace.model.MapDocument; +import eu.dnetlib.pace.util.BlockProcessor; import eu.dnetlib.pace.utils.PaceUtils; import eu.dnetlib.reporter.SparkCounter; import eu.dnetlib.reporter.SparkReporter; @@ -19,11 +19,9 @@ import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.graphx.Edge; import org.apache.spark.rdd.RDD; import scala.Tuple2; -import scala.collection.Iterable; import java.io.IOException; import java.io.StringWriter; -import java.util.Iterator; import java.util.Set; import java.util.stream.Collectors; @@ -39,8 +37,8 @@ public class SparkTest { counter = new SparkCounter(context); final DedupConfig config = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf")); - BlockProcessor.constructAccumulator(config); + BlockProcessor.constructAccumulator(config); BlockProcessor.accumulators.forEach(acc -> { final String[] values = acc.split("::"); @@ -105,13 +103,4 @@ public class SparkTest { } - - - - - - - - - } diff --git a/dnet-dedup-test/src/main/java/eu/dnetlib/reporter/SparkReporter.java b/dnet-dedup-test/src/main/java/eu/dnetlib/reporter/SparkReporter.java index ba95ea5..7125658 100644 --- a/dnet-dedup-test/src/main/java/eu/dnetlib/reporter/SparkReporter.java +++ b/dnet-dedup-test/src/main/java/eu/dnetlib/reporter/SparkReporter.java @@ -1,18 +1,12 @@ package eu.dnetlib.reporter; -import eu.dnetlib.DnetAccumulator; -import eu.dnetlib.Reporter; +import eu.dnetlib.pace.util.Reporter; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.spark.api.java.JavaSparkContext; -import org.glassfish.jersey.internal.util.collection.StringIgnoreCaseKeyComparator; import scala.Tuple2; -import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class SparkReporter implements Reporter { diff --git a/dnet-pace-core/pom.xml b/dnet-pace-core/pom.xml index cb31ed6..d35fd68 100644 --- a/dnet-pace-core/pom.xml +++ b/dnet-pace-core/pom.xml @@ -1,70 +1,60 @@ + + 4.0.0 + eu.dnetlib - dnet45-parent - 1.0.0 - + dnet-dedup + 3.0.0-SNAPSHOT + ../pom.xml - 4.0.0 - eu.dnetlib + dnet-pace-core - jar - 2.6.8-SNAPSHOT - - scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-pace-core/trunk - + jar + edu.cmu secondstring - 1.0.0 com.google.guava guava - 15.0 com.google.code.gson gson - ${google.gson.version} commons-lang commons-lang - ${commons.lang.version} commons-io commons-io - ${commons.io.version} commons-collections commons-collections - ${commons.collections.version} com.googlecode.protobuf-java-format protobuf-java-format - 1.2 org.antlr stringtemplate - 3.2 commons-logging commons-logging - ${commons.logging.version} junit junit - ${junit.version} test + diff --git a/dnet-dedup-test/src/main/java/eu/dnetlib/BlockProcessor.java b/dnet-pace-core/src/main/java/eu/dnetlib/pace/util/BlockProcessor.java similarity index 99% rename from dnet-dedup-test/src/main/java/eu/dnetlib/BlockProcessor.java rename to dnet-pace-core/src/main/java/eu/dnetlib/pace/util/BlockProcessor.java index dfcf16a..a9979f5 100644 --- a/dnet-dedup-test/src/main/java/eu/dnetlib/BlockProcessor.java +++ b/dnet-pace-core/src/main/java/eu/dnetlib/pace/util/BlockProcessor.java @@ -1,4 +1,4 @@ -package eu.dnetlib; +package eu.dnetlib.pace.util; import com.google.common.collect.Lists; import eu.dnetlib.pace.clustering.NGramUtils; diff --git a/dnet-dedup-test/src/main/java/eu/dnetlib/Reporter.java b/dnet-pace-core/src/main/java/eu/dnetlib/pace/util/Reporter.java similarity index 87% rename from dnet-dedup-test/src/main/java/eu/dnetlib/Reporter.java rename to dnet-pace-core/src/main/java/eu/dnetlib/pace/util/Reporter.java index 74cf5c1..10c886c 100644 --- a/dnet-dedup-test/src/main/java/eu/dnetlib/Reporter.java +++ b/dnet-pace-core/src/main/java/eu/dnetlib/pace/util/Reporter.java @@ -1,4 +1,4 @@ -package eu.dnetlib; +package eu.dnetlib.pace.util; import java.io.Serializable; diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..cc74ffd --- /dev/null +++ b/pom.xml @@ -0,0 +1,168 @@ + + + + 4.0.0 + + eu.dnetlib + dnet-dedup + 3.0.0-SNAPSHOT + + pom + + http://www.d-net.research-infrastructures.eu + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + scm:git:https://github.com/dnet-team/dnet-dedup.git + + + + dnet-pace-core + dnet-dedup-test + + + + Redmine + https://issue.openaire.research-infrastructures.eu/projects/openaire + + + + + + dnet-deps + dnet-dependencies + http://maven.research-infrastructures.eu/nexus/content/repositories/dnet-deps + default + + + dnet45-releases + D-Net 45 Releases + http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-releases + default + + true + + + + + + target + target/classes + ${project.artifactId}-${project.version} + target/test-classes + + + org.apache.maven.plugins + maven-compiler-plugin + 3.6.0 + + 1.8 + 1.8 + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.0.0 + + + + + + + + + edu.cmu + secondstring + 1.0.0 + + + org.antlr + stringtemplate + 3.2 + + + com.googlecode.protobuf-java-format + protobuf-java-format + 1.2 + + + eu.dnetlib + dnet-openaire-data-protos + 3.9.3-proto250 + + + com.google.guava + guava + ${google.guava.version} + + + com.google.code.gson + gson + ${google.gson.version} + + + commons-lang + commons-lang + ${commons.lang.version} + + + commons-io + commons-io + ${commons.io.version} + + + commons-collections + commons-collections + ${commons.collections.version} + + + commons-logging + commons-logging + ${commons.logging.version} + + + org.apache.spark + spark-core_2.11 + ${spark.version} + + + org.apache.spark + spark-graphx_2.11 + ${spark.version} + + + junit + junit + ${junit.version} + test + + + + + + UTF-8 + UTF-8 + + 2.2.2 + 15.0 + + 2.2.0 + + 2.6 + 2.4 + 3.2.1 + 1.1.3 + + 4.9 + +