lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

Added First Implementation of Spark Test

This commit is contained in:
Sandro La Bruzzo 2018-10-02 17:07:17 +02:00
parent 476c3d7b07
commit d1c73bcf90
5 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dnet-pace-core/pom.xml
View File

@ -23,7 +23,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${google.guava.version}</version>
<version>15.0</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>

7
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/BlacklistAwareClusteringCombiner.java
View File

@ -22,9 +22,12 @@ public class BlacklistAwareClusteringCombiner extends ClusteringCombiner {
private static final Log log = LogFactory.getLog(BlacklistAwareClusteringCombiner.class);
public static Collection<String> filterAndCombine(final MapDocument a, final Config conf, final Map<String, List<String>> blacklists) {
final Document filtered = new BlacklistAwareClusteringCombiner().filter(a, blacklists);
public static Collection<String> filterAndCombine(final MapDocument a, final Config conf) {
final Document filtered = new BlacklistAwareClusteringCombiner().filter(a, conf.blacklists());
return combine(filtered, conf);
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/config/DedupConfig.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.config;
import java.io.IOException;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -19,7 +20,7 @@ import eu.dnetlib.pace.model.FieldDef;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class DedupConfig implements Config {
public class DedupConfig implements Config, Serializable {
private static final Log log = LogFactory.getLog(DedupConfig.class);

4
dnet-pace-core/src/main/java/eu/dnetlib/pace/model/Field.java
View File

@ -2,10 +2,12 @@ package eu.dnetlib.pace.model;
import eu.dnetlib.pace.config.Type;
import java.io.Serializable;
/**
* The Interface Field.
*/
public interface Field extends Iterable<Field> {
public interface Field extends Iterable<Field>, Serializable {
/**
* Gets the name.

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/model/MapDocument.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.pace.model;
import java.io.Serializable;
import java.util.Map;
import java.util.Set;
@ -10,7 +11,7 @@ import com.google.common.collect.Maps;
/**
* The Class MapDocument.
*/
public class MapDocument implements Document {
public class MapDocument implements Document, Serializable {
/** The identifier. */
private String identifier;