1
0
Fork 0

Merge branch 'master' of code-repo.d4science.org:D-Net/dnet-hadoop

This commit is contained in:
Michele Artini 2020-03-25 16:00:10 +01:00
commit 120e823cd1
32 changed files with 7 additions and 13 deletions

View File

@ -7,7 +7,7 @@
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<artifactId>dhp-dedup</artifactId> <artifactId>dhp-dedup-openaire</artifactId>
<build> <build>
<plugins> <plugins>

View File

@ -56,7 +56,7 @@
<mode>cluster</mode> <mode>cluster</mode>
<name>Update Entity</name> <name>Update Entity</name>
<class>eu.dnetlib.dhp.dedup.SparkUpdateEntity</class> <class>eu.dnetlib.dhp.dedup.SparkUpdateEntity</class>
<jar>dhp-dedup-${projectVersion}.jar</jar> <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
<spark-opts> <spark-opts>
--executor-memory ${sparkExecutorMemory} --executor-memory ${sparkExecutorMemory}
--executor-cores ${sparkExecutorCores} --executor-cores ${sparkExecutorCores}
@ -83,7 +83,7 @@
<mode>cluster</mode> <mode>cluster</mode>
<name>Update Relations</name> <name>Update Relations</name>
<class>eu.dnetlib.dhp.dedup.SparkPropagateRelation</class> <class>eu.dnetlib.dhp.dedup.SparkPropagateRelation</class>
<jar>dhp-dedup-${projectVersion}.jar</jar> <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
<spark-opts> <spark-opts>
--executor-memory ${sparkExecutorMemory} --executor-memory ${sparkExecutorMemory}
--executor-cores ${sparkExecutorCores} --executor-cores ${sparkExecutorCores}

View File

@ -60,7 +60,7 @@
<mode>cluster</mode> <mode>cluster</mode>
<name>Create Similarity Relations</name> <name>Create Similarity Relations</name>
<class>eu.dnetlib.dhp.dedup.SparkCreateSimRels</class> <class>eu.dnetlib.dhp.dedup.SparkCreateSimRels</class>
<jar>dhp-dedup-${projectVersion}.jar</jar> <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
<spark-opts> <spark-opts>
--executor-memory ${sparkExecutorMemory} --executor-memory ${sparkExecutorMemory}
--executor-cores ${sparkExecutorCores} --executor-cores ${sparkExecutorCores}
@ -87,7 +87,7 @@
<mode>cluster</mode> <mode>cluster</mode>
<name>Create Merge Relations</name> <name>Create Merge Relations</name>
<class>eu.dnetlib.dhp.dedup.SparkCreateConnectedComponent</class> <class>eu.dnetlib.dhp.dedup.SparkCreateConnectedComponent</class>
<jar>dhp-dedup-${projectVersion}.jar</jar> <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
<spark-opts> <spark-opts>
--executor-memory ${sparkExecutorMemory} --executor-memory ${sparkExecutorMemory}
--executor-cores ${sparkExecutorCores} --executor-cores ${sparkExecutorCores}
@ -115,7 +115,7 @@
<mode>cluster</mode> <mode>cluster</mode>
<name>Create Dedup Record</name> <name>Create Dedup Record</name>
<class>eu.dnetlib.dhp.dedup.SparkCreateDedupRecord</class> <class>eu.dnetlib.dhp.dedup.SparkCreateDedupRecord</class>
<jar>dhp-dedup-${projectVersion}.jar</jar> <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
<spark-opts> <spark-opts>
--executor-memory ${sparkExecutorMemory} --executor-memory ${sparkExecutorMemory}
--executor-cores ${sparkExecutorCores} --executor-cores ${sparkExecutorCores}

View File

@ -1,15 +1,9 @@
package eu.dnetlib.dhp.dedup.jpath; package eu.dnetlib.dhp.dedup.jpath;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.JsonPath;
import eu.dnetlib.pace.config.DedupConfig; import eu.dnetlib.pace.config.DedupConfig;
import eu.dnetlib.pace.model.MapDocument; import eu.dnetlib.pace.model.MapDocument;
import eu.dnetlib.pace.util.MapDocumentUtil; import eu.dnetlib.pace.util.MapDocumentUtil;
import org.apache.commons.io.IOUtils;
import org.junit.Test; import org.junit.Test;
import java.util.List;
import java.util.Map;
public class JsonPathTest { public class JsonPathTest {

View File

@ -17,7 +17,7 @@
<module>dhp-aggregation</module> <module>dhp-aggregation</module>
<module>dhp-distcp</module> <module>dhp-distcp</module>
<module>dhp-graph-mapper</module> <module>dhp-graph-mapper</module>
<module>dhp-dedup</module> <module>dhp-dedup-openaire</module>
<module>dhp-graph-provision</module> <module>dhp-graph-provision</module>
</modules> </modules>