D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

changed code for the mode of DbClient and also removed the dependency to graph-mapper

This commit is contained in:
Miriam Baglioni 2020-05-04 12:26:17 +02:00
parent 31ea05297d
commit e218360f8a
4 changed files with 13 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dhp-workflows/dhp-blacklist/pom.xml
View File

@ -11,13 +11,6 @@
<artifactId>dhp-blacklist</artifactId> <artifactId>dhp-blacklist</artifactId>
<dependencies> <dependencies>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-graph-mapper</artifactId>
<version>1.1.7-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency> <dependency>
<groupId>eu.dnetlib.dhp</groupId> <groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-common</artifactId> <artifactId>dhp-common</artifactId>

3
dhp-workflows/dhp-blacklist/src/main/java/eu/dnetlib/dhp/blacklist/PrepareMergedRelationJob.java
View File

@ -69,7 +69,8 @@ public class PrepareMergedRelationJob {
relation.createOrReplaceTempView("relation"); relation.createOrReplaceTempView("relation");
spark spark
.sql("Select * from relation " + .sql(
"Select * from relation " +
"where relclass = 'merges' " + "where relclass = 'merges' " +
"and datainfo.deletedbyinference = false") "and datainfo.deletedbyinference = false")
.as(Encoders.bean(Relation.class)) .as(Encoders.bean(Relation.class))

2
dhp-workflows/dhp-blacklist/src/main/java/eu/dnetlib/dhp/blacklist/ReadBlacklistFromDB.java
View File

@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.raw.common.DbClient; import eu.dnetlib.dhp.common.DbClient;
import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.common.ModelSupport;
import eu.dnetlib.dhp.schema.common.RelationInverse; import eu.dnetlib.dhp.schema.common.RelationInverse;
import eu.dnetlib.dhp.schema.oaf.Relation; import eu.dnetlib.dhp.schema.oaf.Relation;

21
dhp-workflows/dhp-blacklist/src/main/java/eu/dnetlib/dhp/blacklist/SparkRemoveBlacklistedRelationJob.java
View File

@ -57,17 +57,16 @@ public class SparkRemoveBlacklistedRelationJob {
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();
runWithSparkSession( runWithSparkSession(
conf, conf,
isSparkSessionManaged, isSparkSessionManaged,
spark -> { spark -> {
removeBlacklistedRelations( removeBlacklistedRelations(
spark, spark,
inputPath, inputPath,
blacklistPath, blacklistPath,
outputPath, outputPath,
mergesPath); mergesPath);
}); });
} }