1
0
Fork 0

refactoring

This commit is contained in:
Miriam Baglioni 2020-12-01 14:31:34 +01:00
parent 62ff4999e3
commit 5b3ed70808
7 changed files with 480 additions and 401 deletions

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import java.io.Serializable; import java.io.Serializable;
@ -5,7 +6,6 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
public class BipDeserialize extends HashMap<String, List<Score>> implements Serializable { public class BipDeserialize extends HashMap<String, List<Score>> implements Serializable {
public BipDeserialize() { public BipDeserialize() {

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import java.io.Serializable; import java.io.Serializable;

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import java.io.Serializable; import java.io.Serializable;

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import java.io.Serializable; import java.io.Serializable;

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import java.io.Serializable; import java.io.Serializable;

View File

@ -1,11 +1,13 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import com.fasterxml.jackson.databind.ObjectMapper; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.HdfsSupport; import java.io.Serializable;
import eu.dnetlib.dhp.schema.action.AtomicAction; import java.util.List;
import eu.dnetlib.dhp.schema.oaf.*; import java.util.Optional;
import eu.dnetlib.dhp.schema.oaf.KeyValue; import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
@ -22,15 +24,16 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.HdfsSupport;
import eu.dnetlib.dhp.schema.action.AtomicAction;
import eu.dnetlib.dhp.schema.oaf.*;
import eu.dnetlib.dhp.schema.oaf.KeyValue;
import scala.Tuple2; import scala.Tuple2;
import java.io.Serializable;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
public class SparkAtomicActionScoreJob implements Serializable { public class SparkAtomicActionScoreJob implements Serializable {
private static String DOI = "doi"; private static String DOI = "doi";
@ -86,18 +89,17 @@ public class SparkAtomicActionScoreJob implements Serializable {
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
JavaRDD<BipDeserialize> bipDeserializeJavaRDD = sc.textFile(bipScorePath) JavaRDD<BipDeserialize> bipDeserializeJavaRDD = sc
.textFile(bipScorePath)
.map(item -> OBJECT_MAPPER.readValue(item, BipDeserialize.class)); .map(item -> OBJECT_MAPPER.readValue(item, BipDeserialize.class));
Dataset<BipScore> bipScores = spark
Dataset<BipScore> bipScores = spark.createDataset(bipDeserializeJavaRDD.flatMap(entry -> .createDataset(bipDeserializeJavaRDD.flatMap(entry -> entry.keySet().stream().map(key -> {
entry.keySet().stream().map(key -> {
BipScore bs = new BipScore(); BipScore bs = new BipScore();
bs.setId(key); bs.setId(key);
bs.setScoreList(entry.get(key)); bs.setScoreList(entry.get(key));
return bs; return bs;
}).collect(Collectors.toList()).iterator() }).collect(Collectors.toList()).iterator()).rdd(), Encoders.bean(BipScore.class));
).rdd(), Encoders.bean(BipScore.class));
System.out.println(bipScores.count()); System.out.println(bipScores.count());
@ -105,13 +107,17 @@ public class SparkAtomicActionScoreJob implements Serializable {
results.createOrReplaceTempView("result"); results.createOrReplaceTempView("result");
Dataset<PreparedResult> preparedResult = spark.sql("select pIde.value value, id " + Dataset<PreparedResult> preparedResult = spark
.sql(
"select pIde.value value, id " +
"from result " + "from result " +
"lateral view explode (pid) p as pIde " + "lateral view explode (pid) p as pIde " +
"where dataInfo.deletedbyinference = false and pIde.qualifier.classid = '" + DOI + "'") "where dataInfo.deletedbyinference = false and pIde.qualifier.classid = '" + DOI + "'")
.as(Encoders.bean(PreparedResult.class)); .as(Encoders.bean(PreparedResult.class));
Dataset<BipScore> tmp = bipScores.joinWith(preparedResult, bipScores.col("id").equalTo(preparedResult.col("value")), Dataset<BipScore> tmp = bipScores
.joinWith(
preparedResult, bipScores.col("id").equalTo(preparedResult.col("value")),
"inner") "inner")
.map((MapFunction<Tuple2<BipScore, PreparedResult>, BipScore>) value -> { .map((MapFunction<Tuple2<BipScore, PreparedResult>, BipScore>) value -> {
BipScore ret = value._1(); BipScore ret = value._1();
@ -119,9 +125,9 @@ public class SparkAtomicActionScoreJob implements Serializable {
return ret; return ret;
}, Encoders.bean(BipScore.class)); }, Encoders.bean(BipScore.class));
tmp.groupByKey((MapFunction<BipScore, String>) value -> value.getId(), Encoders.STRING()) tmp
.mapGroups((MapGroupsFunction<String, BipScore, I>) (k, it) -> .groupByKey((MapFunction<BipScore, String>) value -> value.getId(), Encoders.STRING())
{ .mapGroups((MapGroupsFunction<String, BipScore, I>) (k, it) -> {
Result ret = inputClazz.newInstance(); Result ret = inputClazz.newInstance();
BipScore first = it.next(); BipScore first = it.next();
ret.setId(first.getId()); ret.setId(first.getId());
@ -141,36 +147,40 @@ public class SparkAtomicActionScoreJob implements Serializable {
} }
public static Dataset<BipScore> getBipScoreDataset(Dataset<BipDeserialize> bipdeserialized) { public static Dataset<BipScore> getBipScoreDataset(Dataset<BipDeserialize> bipdeserialized) {
return bipdeserialized.flatMap((FlatMapFunction<BipDeserialize, BipScore>) bip -> return bipdeserialized
bip.keySet().stream().map(key -> { .flatMap((FlatMapFunction<BipDeserialize, BipScore>) bip -> bip.keySet().stream().map(key -> {
BipScore bs = new BipScore(); BipScore bs = new BipScore();
bs.setId(key); bs.setId(key);
bs.setScoreList(bip.get(key)); bs.setScoreList(bip.get(key));
return bs; return bs;
}).collect(Collectors.toList()).iterator() }).collect(Collectors.toList()).iterator(), Encoders.bean(BipScore.class));
, Encoders.bean(BipScore.class));
} }
private static List<Measure> getMeasure(BipScore value) { private static List<Measure> getMeasure(BipScore value) {
return value.getScoreList() return value
.getScoreList()
.stream() .stream()
.map(score -> { .map(score -> {
Measure m = new Measure(); Measure m = new Measure();
m.setId(score.getId()); m.setId(score.getId());
m.setUnit(score.getUnit().stream() m
.setUnit(
score
.getUnit()
.stream()
.map(unit -> { .map(unit -> {
KeyValue kv = new KeyValue(); KeyValue kv = new KeyValue();
kv.setValue(unit.getValue()); kv.setValue(unit.getValue());
kv.setKey(unit.getKey()); kv.setKey(unit.getKey());
kv.setDataInfo(getDataInfo()); kv.setDataInfo(getDataInfo());
return kv; return kv;
}).collect(Collectors.toList())); })
.collect(Collectors.toList()));
return m; return m;
}).collect(Collectors.toList()); })
.collect(Collectors.toList());
} }
private static DataInfo getDataInfo() { private static DataInfo getDataInfo() {
DataInfo di = new DataInfo(); DataInfo di = new DataInfo();
di.setInferred(false); di.setInferred(false);
@ -186,7 +196,6 @@ public class SparkAtomicActionScoreJob implements Serializable {
return di; return di;
} }
private static void removeOutputDir(SparkSession spark, String path) { private static void removeOutputDir(SparkSession spark, String path) {
HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration());
} }
@ -199,5 +208,4 @@ public class SparkAtomicActionScoreJob implements Serializable {
.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); .map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
} }
} }

View File

@ -1,9 +1,12 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException;
import eu.dnetlib.dhp.schema.action.AtomicAction; import java.nio.file.Files;
import eu.dnetlib.dhp.schema.oaf.*; import java.nio.file.Path;
import eu.dnetlib.dhp.schema.oaf.KeyValue; import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat;
@ -24,13 +27,13 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import scala.Tuple2;
import java.io.IOException; import com.fasterxml.jackson.databind.ObjectMapper;
import java.nio.file.Files;
import java.nio.file.Path; import eu.dnetlib.dhp.schema.action.AtomicAction;
import java.util.List; import eu.dnetlib.dhp.schema.oaf.*;
import java.util.stream.Collectors; import eu.dnetlib.dhp.schema.oaf.KeyValue;
import scala.Tuple2;
public class SparkAtomicActionScoreJobTest { public class SparkAtomicActionScoreJobTest {
@ -73,13 +76,16 @@ public class SparkAtomicActionScoreJobTest {
@Test @Test
public void matchOne() throws Exception { public void matchOne() throws Exception {
String bipScoresPath = getClass().getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json").getPath(); String bipScoresPath = getClass()
.getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json")
.getPath();
String inputPath = getClass() String inputPath = getClass()
.getResource( .getResource(
"/eu/dnetlib/dhp/actionmanager/bipfinder/publication.json") "/eu/dnetlib/dhp/actionmanager/bipfinder/publication.json")
.getPath(); .getPath();
SparkAtomicActionScoreJob.main( SparkAtomicActionScoreJob
.main(
new String[] { new String[] {
"-isSparkSessionManaged", "-isSparkSessionManaged",
Boolean.FALSE.toString(), Boolean.FALSE.toString(),
@ -93,7 +99,6 @@ public class SparkAtomicActionScoreJobTest {
workingDir.toString() + "/actionSet" workingDir.toString() + "/actionSet"
}); });
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
JavaRDD<Publication> tmp = sc JavaRDD<Publication> tmp = sc
@ -106,32 +111,43 @@ public class SparkAtomicActionScoreJobTest {
Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class)); Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class));
verificationDataset.createOrReplaceTempView("publication"); verificationDataset.createOrReplaceTempView("publication");
Dataset<Row> execVerification = spark.sql("Select p.id oaid, mes.id, mUnit.value from publication p " + Dataset<Row> execVerification = spark
.sql(
"Select p.id oaid, mes.id, mUnit.value from publication p " +
"lateral view explode(measures) m as mes " + "lateral view explode(measures) m as mes " +
"lateral view explode(mes.unit) u as mUnit "); "lateral view explode(mes.unit) u as mUnit ");
Assertions.assertEquals(2, execVerification.count()); Assertions.assertEquals(2, execVerification.count());
Assertions.assertEquals("50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb", Assertions
.assertEquals(
"50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb",
execVerification.select("oaid").collectAsList().get(0).getString(0)); execVerification.select("oaid").collectAsList().get(0).getString(0));
Assertions.assertEquals("1.47565045883e-08", Assertions
.assertEquals(
"1.47565045883e-08",
execVerification.filter("id = 'influence'").select("value").collectAsList().get(0).getString(0)); execVerification.filter("id = 'influence'").select("value").collectAsList().get(0).getString(0));
Assertions.assertEquals("0.227515392", Assertions
.assertEquals(
"0.227515392",
execVerification.filter("id = 'popularity'").select("value").collectAsList().get(0).getString(0)); execVerification.filter("id = 'popularity'").select("value").collectAsList().get(0).getString(0));
} }
@Test @Test
public void matchOneWithTwo() throws Exception { public void matchOneWithTwo() throws Exception {
String bipScoresPath = getClass().getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json").getPath(); String bipScoresPath = getClass()
.getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json")
.getPath();
String inputPath = getClass() String inputPath = getClass()
.getResource( .getResource(
"/eu/dnetlib/dhp/actionmanager/bipfinder/publication_2.json") "/eu/dnetlib/dhp/actionmanager/bipfinder/publication_2.json")
.getPath(); .getPath();
SparkAtomicActionScoreJob.main( SparkAtomicActionScoreJob
.main(
new String[] { new String[] {
"-isSparkSessionManaged", "-isSparkSessionManaged",
Boolean.FALSE.toString(), Boolean.FALSE.toString(),
@ -145,7 +161,6 @@ public class SparkAtomicActionScoreJobTest {
workingDir.toString() + "/actionSet" workingDir.toString() + "/actionSet"
}); });
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
JavaRDD<Publication> tmp = sc JavaRDD<Publication> tmp = sc
@ -158,45 +173,58 @@ public class SparkAtomicActionScoreJobTest {
Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class)); Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class));
verificationDataset.createOrReplaceTempView("publication"); verificationDataset.createOrReplaceTempView("publication");
Dataset<Row> execVerification = spark.sql("Select p.id oaid, mes.id, mUnit.value from publication p " + Dataset<Row> execVerification = spark
.sql(
"Select p.id oaid, mes.id, mUnit.value from publication p " +
"lateral view explode(measures) m as mes " + "lateral view explode(measures) m as mes " +
"lateral view explode(mes.unit) u as mUnit "); "lateral view explode(mes.unit) u as mUnit ");
Assertions.assertEquals(4, execVerification.count()); Assertions.assertEquals(4, execVerification.count());
Assertions
Assertions.assertEquals("50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb", .assertEquals(
"50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb",
execVerification.select("oaid").collectAsList().get(0).getString(0)); execVerification.select("oaid").collectAsList().get(0).getString(0));
Assertions.assertEquals(2, Assertions
.assertEquals(
2,
execVerification.filter("id = 'influence'").count()); execVerification.filter("id = 'influence'").count());
Assertions.assertEquals(2, Assertions
.assertEquals(
2,
execVerification.filter("id = 'popularity'").count()); execVerification.filter("id = 'popularity'").count());
List<Row> tmp_ds = execVerification.filter("id = 'influence'").select("value").collectAsList(); List<Row> tmp_ds = execVerification.filter("id = 'influence'").select("value").collectAsList();
String tmp_influence = tmp_ds.get(0).getString(0); String tmp_influence = tmp_ds.get(0).getString(0);
Assertions.assertTrue("1.47565045883e-08".equals(tmp_influence)|| Assertions
.assertTrue(
"1.47565045883e-08".equals(tmp_influence) ||
"1.98956540239e-08".equals(tmp_influence)); "1.98956540239e-08".equals(tmp_influence));
tmp_influence = tmp_ds.get(1).getString(0); tmp_influence = tmp_ds.get(1).getString(0);
Assertions.assertTrue("1.47565045883e-08".equals(tmp_influence)|| Assertions
.assertTrue(
"1.47565045883e-08".equals(tmp_influence) ||
"1.98956540239e-08".equals(tmp_influence)); "1.98956540239e-08".equals(tmp_influence));
Assertions.assertTrue(!tmp_ds.get(0).getString(0).equals(tmp_ds.get(1).getString(0))); Assertions.assertTrue(!tmp_ds.get(0).getString(0).equals(tmp_ds.get(1).getString(0)));
} }
@Test @Test
public void matchTwo() throws Exception { public void matchTwo() throws Exception {
String bipScoresPath = getClass().getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json").getPath(); String bipScoresPath = getClass()
.getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json")
.getPath();
String inputPath = getClass() String inputPath = getClass()
.getResource( .getResource(
"/eu/dnetlib/dhp/actionmanager/bipfinder/publication_3.json") "/eu/dnetlib/dhp/actionmanager/bipfinder/publication_3.json")
.getPath(); .getPath();
SparkAtomicActionScoreJob.main( SparkAtomicActionScoreJob
.main(
new String[] { new String[] {
"-isSparkSessionManaged", "-isSparkSessionManaged",
Boolean.FALSE.toString(), Boolean.FALSE.toString(),
@ -210,7 +238,6 @@ public class SparkAtomicActionScoreJobTest {
workingDir.toString() + "/actionSet" workingDir.toString() + "/actionSet"
}); });
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
JavaRDD<Publication> tmp = sc JavaRDD<Publication> tmp = sc
@ -223,41 +250,81 @@ public class SparkAtomicActionScoreJobTest {
Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class)); Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class));
verificationDataset.createOrReplaceTempView("publication"); verificationDataset.createOrReplaceTempView("publication");
Dataset<Row> execVerification = spark.sql("Select p.id oaid, mes.id, mUnit.value from publication p " + Dataset<Row> execVerification = spark
.sql(
"Select p.id oaid, mes.id, mUnit.value from publication p " +
"lateral view explode(measures) m as mes " + "lateral view explode(measures) m as mes " +
"lateral view explode(mes.unit) u as mUnit "); "lateral view explode(mes.unit) u as mUnit ");
Assertions.assertEquals(4, execVerification.count()); Assertions.assertEquals(4, execVerification.count());
Assertions
Assertions.assertEquals(2, .assertEquals(
2,
execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb'").count()); execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb'").count());
Assertions.assertEquals(2, Assertions
.assertEquals(
2,
execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09'").count()); execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09'").count());
Assertions
Assertions.assertEquals(2, .assertEquals(
2,
execVerification.filter("id = 'influence'").count()); execVerification.filter("id = 'influence'").count());
Assertions.assertEquals(2, Assertions
.assertEquals(
2,
execVerification.filter("id = 'popularity'").count()); execVerification.filter("id = 'popularity'").count());
Assertions.assertEquals("1.47565045883e-08" , Assertions
execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " + .assertEquals(
"and id = 'influence'").select("value").collectAsList().get(0).getString(0)); "1.47565045883e-08",
execVerification
.filter(
"oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " +
"and id = 'influence'")
.select("value")
.collectAsList()
.get(0)
.getString(0));
Assertions.assertEquals( Assertions
"1.98956540239e-08", execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " + .assertEquals(
"and id = 'influence'").select("value").collectAsList().get(0).getString(0)); "1.98956540239e-08",
execVerification
.filter(
"oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " +
"and id = 'influence'")
.select("value")
.collectAsList()
.get(0)
.getString(0));
Assertions.assertEquals( Assertions
"0.282046161584", execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " + .assertEquals(
"and id = 'popularity'").select("value").collectAsList().get(0).getString(0)); "0.282046161584",
execVerification
.filter(
"oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " +
"and id = 'popularity'")
.select("value")
.collectAsList()
.get(0)
.getString(0));
Assertions.assertEquals("0.227515392" , Assertions
execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " + .assertEquals(
"and id = 'popularity'").select("value").collectAsList().get(0).getString(0)); "0.227515392",
execVerification
.filter(
"oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " +
"and id = 'popularity'")
.select("value")
.collectAsList()
.get(0)
.getString(0));
} }