From d1c73bcf9049ee7b5b5234598665f0a5393ac1a0 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Tue, 2 Oct 2018 17:07:17 +0200 Subject: [PATCH] Added First Implementation of Spark Test --- dnet-pace-core/pom.xml | 2 +- .../pace/clustering/BlacklistAwareClusteringCombiner.java | 7 +++++-- .../src/main/java/eu/dnetlib/pace/config/DedupConfig.java | 3 ++- .../src/main/java/eu/dnetlib/pace/model/Field.java | 4 +++- .../src/main/java/eu/dnetlib/pace/model/MapDocument.java | 3 ++- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dnet-pace-core/pom.xml b/dnet-pace-core/pom.xml index 8e0e0f741..cb31ed689 100644 --- a/dnet-pace-core/pom.xml +++ b/dnet-pace-core/pom.xml @@ -23,7 +23,7 @@ com.google.guava guava - ${google.guava.version} + 15.0 com.google.code.gson diff --git a/dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/BlacklistAwareClusteringCombiner.java b/dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/BlacklistAwareClusteringCombiner.java index b007853a9..4ecedc4c0 100644 --- a/dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/BlacklistAwareClusteringCombiner.java +++ b/dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/BlacklistAwareClusteringCombiner.java @@ -22,9 +22,12 @@ public class BlacklistAwareClusteringCombiner extends ClusteringCombiner { private static final Log log = LogFactory.getLog(BlacklistAwareClusteringCombiner.class); - public static Collection filterAndCombine(final MapDocument a, final Config conf, final Map> blacklists) { - final Document filtered = new BlacklistAwareClusteringCombiner().filter(a, blacklists); + + + public static Collection filterAndCombine(final MapDocument a, final Config conf) { + + final Document filtered = new BlacklistAwareClusteringCombiner().filter(a, conf.blacklists()); return combine(filtered, conf); } diff --git a/dnet-pace-core/src/main/java/eu/dnetlib/pace/config/DedupConfig.java b/dnet-pace-core/src/main/java/eu/dnetlib/pace/config/DedupConfig.java index 5116f3684..eb041844a 100644 --- a/dnet-pace-core/src/main/java/eu/dnetlib/pace/config/DedupConfig.java +++ b/dnet-pace-core/src/main/java/eu/dnetlib/pace/config/DedupConfig.java @@ -1,6 +1,7 @@ package eu.dnetlib.pace.config; import java.io.IOException; +import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -19,7 +20,7 @@ import eu.dnetlib.pace.model.FieldDef; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -public class DedupConfig implements Config { +public class DedupConfig implements Config, Serializable { private static final Log log = LogFactory.getLog(DedupConfig.class); diff --git a/dnet-pace-core/src/main/java/eu/dnetlib/pace/model/Field.java b/dnet-pace-core/src/main/java/eu/dnetlib/pace/model/Field.java index 9c97ce38d..4b7a73e31 100644 --- a/dnet-pace-core/src/main/java/eu/dnetlib/pace/model/Field.java +++ b/dnet-pace-core/src/main/java/eu/dnetlib/pace/model/Field.java @@ -2,10 +2,12 @@ package eu.dnetlib.pace.model; import eu.dnetlib.pace.config.Type; +import java.io.Serializable; + /** * The Interface Field. */ -public interface Field extends Iterable { +public interface Field extends Iterable, Serializable { /** * Gets the name. diff --git a/dnet-pace-core/src/main/java/eu/dnetlib/pace/model/MapDocument.java b/dnet-pace-core/src/main/java/eu/dnetlib/pace/model/MapDocument.java index 74935de56..77b7c120e 100644 --- a/dnet-pace-core/src/main/java/eu/dnetlib/pace/model/MapDocument.java +++ b/dnet-pace-core/src/main/java/eu/dnetlib/pace/model/MapDocument.java @@ -1,5 +1,6 @@ package eu.dnetlib.pace.model; +import java.io.Serializable; import java.util.Map; import java.util.Set; @@ -10,7 +11,7 @@ import com.google.common.collect.Maps; /** * The Class MapDocument. */ -public class MapDocument implements Document { +public class MapDocument implements Document, Serializable { /** The identifier. */ private String identifier;