forked from D-Net/dnet-hadoop
changed code for the mode of DbClient and also removed the dependency to graph-mapper
This commit is contained in:
parent
31ea05297d
commit
e218360f8a
|
@ -11,13 +11,6 @@
|
|||
|
||||
<artifactId>dhp-blacklist</artifactId>
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-graph-mapper</artifactId>
|
||||
<version>1.1.7-SNAPSHOT</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-common</artifactId>
|
||||
|
|
|
@ -69,7 +69,8 @@ public class PrepareMergedRelationJob {
|
|||
relation.createOrReplaceTempView("relation");
|
||||
|
||||
spark
|
||||
.sql("Select * from relation " +
|
||||
.sql(
|
||||
"Select * from relation " +
|
||||
"where relclass = 'merges' " +
|
||||
"and datainfo.deletedbyinference = false")
|
||||
.as(Encoders.bean(Relation.class))
|
||||
|
|
|
@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path;
|
|||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
import eu.dnetlib.dhp.oa.graph.raw.common.DbClient;
|
||||
import eu.dnetlib.dhp.common.DbClient;
|
||||
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
||||
import eu.dnetlib.dhp.schema.common.RelationInverse;
|
||||
import eu.dnetlib.dhp.schema.oaf.Relation;
|
||||
|
|
|
@ -57,17 +57,16 @@ public class SparkRemoveBlacklistedRelationJob {
|
|||
SparkConf conf = new SparkConf();
|
||||
|
||||
runWithSparkSession(
|
||||
conf,
|
||||
isSparkSessionManaged,
|
||||
spark -> {
|
||||
removeBlacklistedRelations(
|
||||
spark,
|
||||
inputPath,
|
||||
blacklistPath,
|
||||
outputPath,
|
||||
mergesPath);
|
||||
});
|
||||
|
||||
conf,
|
||||
isSparkSessionManaged,
|
||||
spark -> {
|
||||
removeBlacklistedRelations(
|
||||
spark,
|
||||
inputPath,
|
||||
blacklistPath,
|
||||
outputPath,
|
||||
mergesPath);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue