spyros
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

implementation of the wf to dedup entities, addition of the module to run the wf on the cluster

This commit is contained in:
miconis 2020-12-04 15:41:31 +01:00
parent 72116446ec
commit ed0d5d3e1d
6 changed files with 23 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dnet-pace-core/pom.xml
View File

@ -43,9 +43,10 @@
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>

11
dnet-pace-core/src/test/java/eu/dnetlib/pace/clustering/ClusteringFunctionTest.java
View File

@ -5,18 +5,17 @@ import com.google.common.collect.Maps;
import eu.dnetlib.pace.AbstractPaceTest;
import eu.dnetlib.pace.common.AbstractPaceFunctions;
import eu.dnetlib.pace.config.DedupConfig;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.*;
import java.util.Map;
public class ClusteringFunctionTest extends AbstractPaceTest {
private Map<String, Integer> params;
DedupConfig conf;
private static Map<String, Integer> params;
private static DedupConfig conf;
@Before
public void setUp() throws Exception {
@BeforeAll
public static void setUp() throws Exception {
params = Maps.newHashMap();
conf = DedupConfig.load(AbstractPaceFunctions.readFromClasspath("/eu/dnetlib/pace/config/organization.current.conf.json", ClusteringFunctionTest.class));
}

10
dnet-pace-core/src/test/java/eu/dnetlib/pace/common/PaceFunctionTest.java
View File

@ -1,10 +1,8 @@
package eu.dnetlib.pace.common;
import org.junit.Assert;
import org.junit.Test;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
import org.junit.jupiter.api.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class PaceFunctionTest extends AbstractPaceFunctions {
@ -27,7 +25,7 @@ public class PaceFunctionTest extends AbstractPaceFunctions {
@Test
public void normalizeTest() {
Assert.assertEquals("universitat", normalize("Universität"));
assertEquals("universitat", normalize("Universität"));
System.out.println(normalize("İstanbul Ticarət Universiteti"));
}

9
dnet-pace-core/src/test/java/eu/dnetlib/pace/comparators/ComparatorTest.java
View File

@ -5,23 +5,22 @@ import eu.dnetlib.pace.model.MapDocument;
import eu.dnetlib.pace.tree.*;
import eu.dnetlib.pace.config.DedupConfig;
import eu.dnetlib.pace.util.MapDocumentUtil;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import eu.dnetlib.pace.common.AbstractPaceFunctions;
import java.util.HashMap;
import java.util.Map;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
public class ComparatorTest extends AbstractPaceFunctions {
private Map<String, String> params;
private DedupConfig conf;
@Before
@BeforeAll
public void setup() {
params = new HashMap<>();
params.put("weight", "1.0");

10
dnet-pace-core/src/test/java/eu/dnetlib/pace/config/ConfigTest.java
View File

@ -7,22 +7,22 @@ import eu.dnetlib.pace.model.FieldList;
import eu.dnetlib.pace.model.MapDocument;
import eu.dnetlib.pace.tree.JsonListMatch;
import eu.dnetlib.pace.util.MapDocumentUtil;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
public class ConfigTest extends AbstractPaceTest {
private Map<String, String> params;
@Before
@BeforeAll
public void setup() {
params = new HashMap<>();
params.put("jpath_value", "$.value");

5
dnet-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.pace.util;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.*;
import java.util.HashMap;
import java.util.Map;
@ -10,7 +9,7 @@ public class UtilTest {
Map<String, String> params;
@Before
@BeforeAll
public void setUp(){
params = new HashMap<String, String>();
}