D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 8 Projects 2 Releases 1 Wiki Activity

implementation of the wf to dedup entities, addition of the module to run the wf on the cluster

This commit is contained in:
miconis 2020-12-04 15:41:31 +01:00
parent 72116446ec
commit ed0d5d3e1d
6 changed files with 23 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dnet-pace-core/pom.xml
View File

@ -43,9 +43,10 @@
<groupId>commons-logging</groupId> <groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId> <artifactId>commons-logging</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>org.junit.jupiter</groupId>
<artifactId>junit</artifactId> <artifactId>junit-jupiter</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>

11
dnet-pace-core/src/test/java/eu/dnetlib/pace/clustering/ClusteringFunctionTest.java
View File

@ -5,18 +5,17 @@ import com.google.common.collect.Maps;
import eu.dnetlib.pace.AbstractPaceTest; import eu.dnetlib.pace.AbstractPaceTest;
import eu.dnetlib.pace.common.AbstractPaceFunctions; import eu.dnetlib.pace.common.AbstractPaceFunctions;
import eu.dnetlib.pace.config.DedupConfig; import eu.dnetlib.pace.config.DedupConfig;
import org.junit.Before; import org.junit.jupiter.api.*;
import org.junit.Test;
import java.util.Map; import java.util.Map;
public class ClusteringFunctionTest extends AbstractPaceTest { public class ClusteringFunctionTest extends AbstractPaceTest {
private Map<String, Integer> params; private static Map<String, Integer> params;
DedupConfig conf; private static DedupConfig conf;
@Before @BeforeAll
public void setUp() throws Exception { public static void setUp() throws Exception {
params = Maps.newHashMap(); params = Maps.newHashMap();
conf = DedupConfig.load(AbstractPaceFunctions.readFromClasspath("/eu/dnetlib/pace/config/organization.current.conf.json", ClusteringFunctionTest.class)); conf = DedupConfig.load(AbstractPaceFunctions.readFromClasspath("/eu/dnetlib/pace/config/organization.current.conf.json", ClusteringFunctionTest.class));
} }

10
dnet-pace-core/src/test/java/eu/dnetlib/pace/common/PaceFunctionTest.java
View File

@ -1,10 +1,8 @@
package eu.dnetlib.pace.common; package eu.dnetlib.pace.common;
import org.junit.Assert; import org.junit.jupiter.api.*;
import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
public class PaceFunctionTest extends AbstractPaceFunctions { public class PaceFunctionTest extends AbstractPaceFunctions {
@ -27,7 +25,7 @@ public class PaceFunctionTest extends AbstractPaceFunctions {
@Test @Test
public void normalizeTest() { public void normalizeTest() {
Assert.assertEquals("universitat", normalize("Universität")); assertEquals("universitat", normalize("Universität"));
System.out.println(normalize("İstanbul Ticarət Universiteti")); System.out.println(normalize("İstanbul Ticarət Universiteti"));
} }

9
dnet-pace-core/src/test/java/eu/dnetlib/pace/comparators/ComparatorTest.java
View File

@ -5,23 +5,22 @@ import eu.dnetlib.pace.model.MapDocument;
import eu.dnetlib.pace.tree.*; import eu.dnetlib.pace.tree.*;
import eu.dnetlib.pace.config.DedupConfig; import eu.dnetlib.pace.config.DedupConfig;
import eu.dnetlib.pace.util.MapDocumentUtil; import eu.dnetlib.pace.util.MapDocumentUtil;
import org.junit.Before;
import org.junit.Test; import org.junit.jupiter.api.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import eu.dnetlib.pace.common.AbstractPaceFunctions; import eu.dnetlib.pace.common.AbstractPaceFunctions;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
public class ComparatorTest extends AbstractPaceFunctions { public class ComparatorTest extends AbstractPaceFunctions {
private Map<String, String> params; private Map<String, String> params;
private DedupConfig conf; private DedupConfig conf;
@Before @BeforeAll
public void setup() { public void setup() {
params = new HashMap<>(); params = new HashMap<>();
params.put("weight", "1.0"); params.put("weight", "1.0");

10
dnet-pace-core/src/test/java/eu/dnetlib/pace/config/ConfigTest.java
View File

@ -7,22 +7,22 @@ import eu.dnetlib.pace.model.FieldList;
import eu.dnetlib.pace.model.MapDocument; import eu.dnetlib.pace.model.MapDocument;
import eu.dnetlib.pace.tree.JsonListMatch; import eu.dnetlib.pace.tree.JsonListMatch;
import eu.dnetlib.pace.util.MapDocumentUtil; import eu.dnetlib.pace.util.MapDocumentUtil;
import org.junit.Before; import org.junit.jupiter.api.*;
import org.junit.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
public class ConfigTest extends AbstractPaceTest { public class ConfigTest extends AbstractPaceTest {
private Map<String, String> params; private Map<String, String> params;
@Before @BeforeAll
public void setup() { public void setup() {
params = new HashMap<>(); params = new HashMap<>();
params.put("jpath_value", "$.value"); params.put("jpath_value", "$.value");

5
dnet-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.pace.util; package eu.dnetlib.pace.util;
import org.junit.Before; import org.junit.jupiter.api.*;
import org.junit.Test;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -10,7 +9,7 @@ public class UtilTest {
Map<String, String> params; Map<String, String> params;
@Before @BeforeAll
public void setUp(){ public void setUp(){
params = new HashMap<String, String>(); params = new HashMap<String, String>();
} }