D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 9 Projects 2 Releases 1 Wiki Activity

refactoring

This commit is contained in:
Miriam Baglioni 2020-12-01 14:31:34 +01:00
parent 62ff4999e3
commit 5b3ed70808
7 changed files with 480 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/BipDeserialize.java
View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import java.io.Serializable; import java.io.Serializable;
@ -5,19 +6,18 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
public class BipDeserialize extends HashMap<String, List<Score>> implements Serializable {
public class BipDeserialize extends HashMap<String, List<Score>> implements Serializable { public BipDeserialize() {
super();
}
public BipDeserialize(){ public List<Score> get(String key) {
super();
}
public List<Score> get(String key) { if (super.get(key) == null) {
return new ArrayList<>();
if (super.get(key) == null) { }
return new ArrayList<>(); return super.get(key);
} }
return super.get(key);
}
} }

29
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/BipScore.java
View File

@ -1,25 +1,26 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import java.io.Serializable; import java.io.Serializable;
import java.util.List; import java.util.List;
public class BipScore implements Serializable { public class BipScore implements Serializable {
private String id; private String id;
private List<Score> scoreList; private List<Score> scoreList;
public String getId() { public String getId() {
return id; return id;
} }
public void setId(String id) { public void setId(String id) {
this.id = id; this.id = id;
} }
public List<Score> getScoreList() { public List<Score> getScoreList() {
return scoreList; return scoreList;
} }
public void setScoreList(List<Score> scoreList) { public void setScoreList(List<Score> scoreList) {
this.scoreList = scoreList; this.scoreList = scoreList;
} }
} }

29
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/KeyValue.java
View File

@ -1,25 +1,26 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import java.io.Serializable; import java.io.Serializable;
public class KeyValue implements Serializable { public class KeyValue implements Serializable {
private String key; private String key;
private String value; private String value;
public String getKey() { public String getKey() {
return key; return key;
} }
public void setKey(String key) { public void setKey(String key) {
this.key = key; this.key = key;
} }
public String getValue() { public String getValue() {
return value; return value;
} }
public void setValue(String value) { public void setValue(String value) {
this.value = value; this.value = value;
} }
} }

29
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/PreparedResult.java
View File

@ -1,24 +1,25 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import java.io.Serializable; import java.io.Serializable;
public class PreparedResult implements Serializable { public class PreparedResult implements Serializable {
private String id; //openaire id private String id; // openaire id
private String value; //doi private String value; // doi
public String getId() { public String getId() {
return id; return id;
} }
public void setId(String id) { public void setId(String id) {
this.id = id; this.id = id;
} }
public String getValue() { public String getValue() {
return value; return value;
} }
public void setValue(String value) { public void setValue(String value) {
this.value = value; this.value = value;
} }
} }

29
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/Score.java
View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import java.io.Serializable; import java.io.Serializable;
@ -5,22 +6,22 @@ import java.util.List;
public class Score implements Serializable { public class Score implements Serializable {
private String id; private String id;
private List<KeyValue> unit; private List<KeyValue> unit;
public String getId() { public String getId() {
return id; return id;
} }
public void setId(String id) { public void setId(String id) {
this.id = id; this.id = id;
} }
public List<KeyValue> getUnit() { public List<KeyValue> getUnit() {
return unit; return unit;
} }
public void setUnit(List<KeyValue> unit) { public void setUnit(List<KeyValue> unit) {
this.unit = unit; this.unit = unit;
} }
} }

306
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java
View File

@ -1,11 +1,13 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import com.fasterxml.jackson.databind.ObjectMapper; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.HdfsSupport; import java.io.Serializable;
import eu.dnetlib.dhp.schema.action.AtomicAction; import java.util.List;
import eu.dnetlib.dhp.schema.oaf.*; import java.util.Optional;
import eu.dnetlib.dhp.schema.oaf.KeyValue; import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
@ -22,182 +24,188 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.HdfsSupport;
import eu.dnetlib.dhp.schema.action.AtomicAction;
import eu.dnetlib.dhp.schema.oaf.*;
import eu.dnetlib.dhp.schema.oaf.KeyValue;
import scala.Tuple2; import scala.Tuple2;
import java.io.Serializable;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
public class SparkAtomicActionScoreJob implements Serializable { public class SparkAtomicActionScoreJob implements Serializable {
private static String DOI = "doi"; private static String DOI = "doi";
private static final Logger log = LoggerFactory.getLogger(SparkAtomicActionScoreJob.class); private static final Logger log = LoggerFactory.getLogger(SparkAtomicActionScoreJob.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
public static <I extends Result> void main(String[] args) throws Exception { public static <I extends Result> void main(String[] args) throws Exception {
String jsonConfiguration = IOUtils String jsonConfiguration = IOUtils
.toString( .toString(
SparkAtomicActionScoreJob.class SparkAtomicActionScoreJob.class
.getResourceAsStream( .getResourceAsStream(
"/eu/dnetlib/dhp/actionmanager/bipfinder/input_parameters.json")); "/eu/dnetlib/dhp/actionmanager/bipfinder/input_parameters.json"));
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
parser.parseArgument(args); parser.parseArgument(args);
Boolean isSparkSessionManaged = Optional Boolean isSparkSessionManaged = Optional
.ofNullable(parser.get("isSparkSessionManaged")) .ofNullable(parser.get("isSparkSessionManaged"))
.map(Boolean::valueOf) .map(Boolean::valueOf)
.orElse(Boolean.TRUE); .orElse(Boolean.TRUE);
log.info("isSparkSessionManaged: {}", isSparkSessionManaged); log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
final String inputPath = parser.get("inputPath"); final String inputPath = parser.get("inputPath");
log.info("inputPath {}: ", inputPath); log.info("inputPath {}: ", inputPath);
final String outputPath = parser.get("outputPath"); final String outputPath = parser.get("outputPath");
log.info("outputPath {}: ", outputPath); log.info("outputPath {}: ", outputPath);
final String bipScorePath = parser.get("bipScorePath"); final String bipScorePath = parser.get("bipScorePath");
log.info("bipScorePath: {}", bipScorePath); log.info("bipScorePath: {}", bipScorePath);
final String resultClassName = parser.get("resultTableName"); final String resultClassName = parser.get("resultTableName");
log.info("resultTableName: {}", resultClassName); log.info("resultTableName: {}", resultClassName);
Class<I> inputClazz = (Class<I>) Class.forName(resultClassName); Class<I> inputClazz = (Class<I>) Class.forName(resultClassName);
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();
runWithSparkSession( runWithSparkSession(
conf, conf,
isSparkSessionManaged, isSparkSessionManaged,
spark -> { spark -> {
removeOutputDir(spark, outputPath); removeOutputDir(spark, outputPath);
prepareResults(spark, inputPath, outputPath, bipScorePath, inputClazz); prepareResults(spark, inputPath, outputPath, bipScorePath, inputClazz);
}); });
} }
private static <I extends Result> void prepareResults(SparkSession spark, String inputPath, String outputPath, private static <I extends Result> void prepareResults(SparkSession spark, String inputPath, String outputPath,
String bipScorePath, Class<I> inputClazz) { String bipScorePath, Class<I> inputClazz) {
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
JavaRDD<BipDeserialize> bipDeserializeJavaRDD = sc.textFile(bipScorePath) JavaRDD<BipDeserialize> bipDeserializeJavaRDD = sc
.map(item -> OBJECT_MAPPER.readValue(item, BipDeserialize.class)); .textFile(bipScorePath)
.map(item -> OBJECT_MAPPER.readValue(item, BipDeserialize.class));
Dataset<BipScore> bipScores = spark
.createDataset(bipDeserializeJavaRDD.flatMap(entry -> entry.keySet().stream().map(key -> {
BipScore bs = new BipScore();
bs.setId(key);
bs.setScoreList(entry.get(key));
return bs;
}).collect(Collectors.toList()).iterator()).rdd(), Encoders.bean(BipScore.class));
Dataset<BipScore> bipScores = spark.createDataset(bipDeserializeJavaRDD.flatMap(entry -> System.out.println(bipScores.count());
entry.keySet().stream().map(key -> {
BipScore bs = new BipScore();
bs.setId(key);
bs.setScoreList(entry.get(key));
return bs;
}).collect(Collectors.toList()).iterator()
).rdd(), Encoders.bean(BipScore.class));
System.out.println(bipScores.count()); Dataset<I> results = readPath(spark, inputPath, inputClazz);
Dataset<I> results = readPath(spark, inputPath, inputClazz); results.createOrReplaceTempView("result");
results.createOrReplaceTempView("result"); Dataset<PreparedResult> preparedResult = spark
.sql(
"select pIde.value value, id " +
"from result " +
"lateral view explode (pid) p as pIde " +
"where dataInfo.deletedbyinference = false and pIde.qualifier.classid = '" + DOI + "'")
.as(Encoders.bean(PreparedResult.class));
Dataset<PreparedResult> preparedResult = spark.sql("select pIde.value value, id " + Dataset<BipScore> tmp = bipScores
"from result " + .joinWith(
"lateral view explode (pid) p as pIde " + preparedResult, bipScores.col("id").equalTo(preparedResult.col("value")),
"where dataInfo.deletedbyinference = false and pIde.qualifier.classid = '" + DOI + "'") "inner")
.as(Encoders.bean(PreparedResult.class)); .map((MapFunction<Tuple2<BipScore, PreparedResult>, BipScore>) value -> {
BipScore ret = value._1();
ret.setId(value._2().getId());
return ret;
}, Encoders.bean(BipScore.class));
Dataset<BipScore> tmp = bipScores.joinWith(preparedResult, bipScores.col("id").equalTo(preparedResult.col("value")), tmp
"inner") .groupByKey((MapFunction<BipScore, String>) value -> value.getId(), Encoders.STRING())
.map((MapFunction<Tuple2<BipScore, PreparedResult>, BipScore>) value -> { .mapGroups((MapGroupsFunction<String, BipScore, I>) (k, it) -> {
BipScore ret = value._1(); Result ret = inputClazz.newInstance();
ret.setId(value._2().getId()); BipScore first = it.next();
return ret; ret.setId(first.getId());
}, Encoders.bean(BipScore.class));
tmp.groupByKey((MapFunction<BipScore, String>) value -> value.getId(), Encoders.STRING()) ret.setMeasures(getMeasure(first));
.mapGroups((MapGroupsFunction<String, BipScore, I>) (k, it) -> it.forEachRemaining(value -> ret.getMeasures().addAll(getMeasure(value)));
{
Result ret = inputClazz.newInstance();
BipScore first = it.next();
ret.setId(first.getId());
ret.setMeasures(getMeasure(first)); return (I) ret;
it.forEachRemaining(value -> ret.getMeasures().addAll(getMeasure(value))); }, Encoders.bean(inputClazz))
.toJavaRDD()
.map(p -> new AtomicAction(inputClazz, p))
.mapToPair(
aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
new Text(OBJECT_MAPPER.writeValueAsString(aa))))
.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class);
;
}
return (I)ret; public static Dataset<BipScore> getBipScoreDataset(Dataset<BipDeserialize> bipdeserialized) {
}, Encoders.bean(inputClazz)) return bipdeserialized
.toJavaRDD() .flatMap((FlatMapFunction<BipDeserialize, BipScore>) bip -> bip.keySet().stream().map(key -> {
.map(p -> new AtomicAction(inputClazz, p)) BipScore bs = new BipScore();
.mapToPair( bs.setId(key);
aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), bs.setScoreList(bip.get(key));
new Text(OBJECT_MAPPER.writeValueAsString(aa)))) return bs;
.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); }).collect(Collectors.toList()).iterator(), Encoders.bean(BipScore.class));
; }
}
public static Dataset<BipScore> getBipScoreDataset(Dataset<BipDeserialize> bipdeserialized){ private static List<Measure> getMeasure(BipScore value) {
return bipdeserialized.flatMap((FlatMapFunction<BipDeserialize, BipScore>) bip -> return value
bip.keySet().stream().map(key -> { .getScoreList()
BipScore bs = new BipScore(); .stream()
bs.setId(key); .map(score -> {
bs.setScoreList(bip.get(key)); Measure m = new Measure();
return bs; m.setId(score.getId());
}).collect(Collectors.toList()).iterator() m
, Encoders.bean(BipScore.class)); .setUnit(
} score
.getUnit()
.stream()
.map(unit -> {
KeyValue kv = new KeyValue();
kv.setValue(unit.getValue());
kv.setKey(unit.getKey());
kv.setDataInfo(getDataInfo());
return kv;
})
.collect(Collectors.toList()));
return m;
})
.collect(Collectors.toList());
}
private static List<Measure> getMeasure(BipScore value) { private static DataInfo getDataInfo() {
return value.getScoreList() DataInfo di = new DataInfo();
.stream() di.setInferred(false);
.map(score -> { di.setInvisible(false);
Measure m = new Measure(); di.setDeletedbyinference(false);
m.setId(score.getId()); di.setTrust("");
m.setUnit(score.getUnit().stream() Qualifier qualifier = new Qualifier();
.map(unit -> { qualifier.setClassid("sysimport:actionset");
KeyValue kv = new KeyValue(); qualifier.setClassname("Harvested");
kv.setValue(unit.getValue()); qualifier.setSchemename("dnet:provenanceActions");
kv.setKey(unit.getKey()); qualifier.setSchemeid("dnet:provenanceActions");
kv.setDataInfo(getDataInfo()); di.setProvenanceaction(qualifier);
return kv; return di;
}).collect(Collectors.toList())); }
return m;
}).collect(Collectors.toList());
}
private static void removeOutputDir(SparkSession spark, String path) {
HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration());
}
public static <R> Dataset<R> readPath(
private static DataInfo getDataInfo() { SparkSession spark, String inputPath, Class<R> clazz) {
DataInfo di = new DataInfo(); return spark
di.setInferred(false); .read()
di.setInvisible(false); .textFile(inputPath)
di.setDeletedbyinference(false); .map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
di.setTrust(""); }
Qualifier qualifier = new Qualifier();
qualifier.setClassid("sysimport:actionset");
qualifier.setClassname("Harvested");
qualifier.setSchemename("dnet:provenanceActions");
qualifier.setSchemeid("dnet:provenanceActions");
di.setProvenanceaction(qualifier);
return di;
}
private static void removeOutputDir(SparkSession spark, String path) {
HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration());
}
public static <R> Dataset<R> readPath(
SparkSession spark, String inputPath, Class<R> clazz) {
return spark
.read()
.textFile(inputPath)
.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
}
} }

437
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJobTest.java
View File

@ -1,9 +1,12 @@
package eu.dnetlib.dhp.actionmanager.bipfinder; package eu.dnetlib.dhp.actionmanager.bipfinder;
import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException;
import eu.dnetlib.dhp.schema.action.AtomicAction; import java.nio.file.Files;
import eu.dnetlib.dhp.schema.oaf.*; import java.nio.file.Path;
import eu.dnetlib.dhp.schema.oaf.KeyValue; import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat;
@ -24,241 +27,305 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import scala.Tuple2;
import java.io.IOException; import com.fasterxml.jackson.databind.ObjectMapper;
import java.nio.file.Files;
import java.nio.file.Path; import eu.dnetlib.dhp.schema.action.AtomicAction;
import java.util.List; import eu.dnetlib.dhp.schema.oaf.*;
import java.util.stream.Collectors; import eu.dnetlib.dhp.schema.oaf.KeyValue;
import scala.Tuple2;
public class SparkAtomicActionScoreJobTest { public class SparkAtomicActionScoreJobTest {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static SparkSession spark; private static SparkSession spark;
private static Path workingDir; private static Path workingDir;
private static final Logger log = LoggerFactory private static final Logger log = LoggerFactory
.getLogger(SparkAtomicActionScoreJobTest.class); .getLogger(SparkAtomicActionScoreJobTest.class);
@BeforeAll @BeforeAll
public static void beforeAll() throws IOException { public static void beforeAll() throws IOException {
workingDir = Files workingDir = Files
.createTempDirectory(SparkAtomicActionScoreJobTest.class.getSimpleName()); .createTempDirectory(SparkAtomicActionScoreJobTest.class.getSimpleName());
log.info("using work dir {}", workingDir); log.info("using work dir {}", workingDir);
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();
conf.setAppName(SparkAtomicActionScoreJobTest.class.getSimpleName()); conf.setAppName(SparkAtomicActionScoreJobTest.class.getSimpleName());
conf.setMaster("local[*]"); conf.setMaster("local[*]");
conf.set("spark.driver.host", "localhost"); conf.set("spark.driver.host", "localhost");
conf.set("hive.metastore.local", "true"); conf.set("hive.metastore.local", "true");
conf.set("spark.ui.enabled", "false"); conf.set("spark.ui.enabled", "false");
conf.set("spark.sql.warehouse.dir", workingDir.toString()); conf.set("spark.sql.warehouse.dir", workingDir.toString());
conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
spark = SparkSession spark = SparkSession
.builder() .builder()
.appName(SparkAtomicActionScoreJobTest.class.getSimpleName()) .appName(SparkAtomicActionScoreJobTest.class.getSimpleName())
.config(conf) .config(conf)
.getOrCreate(); .getOrCreate();
} }
@AfterAll @AfterAll
public static void afterAll() throws IOException { public static void afterAll() throws IOException {
FileUtils.deleteDirectory(workingDir.toFile()); FileUtils.deleteDirectory(workingDir.toFile());
spark.stop(); spark.stop();
} }
@Test @Test
public void matchOne() throws Exception { public void matchOne() throws Exception {
String bipScoresPath = getClass().getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json").getPath(); String bipScoresPath = getClass()
String inputPath = getClass() .getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json")
.getResource( .getPath();
"/eu/dnetlib/dhp/actionmanager/bipfinder/publication.json") String inputPath = getClass()
.getPath(); .getResource(
"/eu/dnetlib/dhp/actionmanager/bipfinder/publication.json")
.getPath();
SparkAtomicActionScoreJob.main( SparkAtomicActionScoreJob
new String[] { .main(
"-isSparkSessionManaged", new String[] {
Boolean.FALSE.toString(), "-isSparkSessionManaged",
"-inputPath", Boolean.FALSE.toString(),
inputPath, "-inputPath",
"-bipScorePath", inputPath,
bipScoresPath, "-bipScorePath",
"-resultTableName", bipScoresPath,
"eu.dnetlib.dhp.schema.oaf.Publication", "-resultTableName",
"-outputPath", "eu.dnetlib.dhp.schema.oaf.Publication",
workingDir.toString() + "/actionSet" "-outputPath",
}); workingDir.toString() + "/actionSet"
});
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); JavaRDD<Publication> tmp = sc
.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
.map(aa -> ((Publication) aa.getPayload()));
JavaRDD<Publication> tmp = sc Assertions.assertTrue(tmp.count() == 1);
.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
.map(aa -> ((Publication) aa.getPayload()));
Assertions.assertTrue(tmp.count() == 1); Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class));
verificationDataset.createOrReplaceTempView("publication");
Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class)); Dataset<Row> execVerification = spark
verificationDataset.createOrReplaceTempView("publication"); .sql(
"Select p.id oaid, mes.id, mUnit.value from publication p " +
"lateral view explode(measures) m as mes " +
"lateral view explode(mes.unit) u as mUnit ");
Dataset<Row> execVerification = spark.sql("Select p.id oaid, mes.id, mUnit.value from publication p " + Assertions.assertEquals(2, execVerification.count());
"lateral view explode(measures) m as mes " +
"lateral view explode(mes.unit) u as mUnit ");
Assertions.assertEquals(2, execVerification.count()); Assertions
.assertEquals(
"50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb",
execVerification.select("oaid").collectAsList().get(0).getString(0));
Assertions.assertEquals("50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb", Assertions
execVerification.select("oaid").collectAsList().get(0).getString(0)); .assertEquals(
"1.47565045883e-08",
execVerification.filter("id = 'influence'").select("value").collectAsList().get(0).getString(0));
Assertions.assertEquals("1.47565045883e-08", Assertions
execVerification.filter("id = 'influence'").select("value").collectAsList().get(0).getString(0)); .assertEquals(
"0.227515392",
execVerification.filter("id = 'popularity'").select("value").collectAsList().get(0).getString(0));
Assertions.assertEquals("0.227515392", }
execVerification.filter("id = 'popularity'").select("value").collectAsList().get(0).getString(0));
} @Test
public void matchOneWithTwo() throws Exception {
String bipScoresPath = getClass()
.getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json")
.getPath();
String inputPath = getClass()
.getResource(
"/eu/dnetlib/dhp/actionmanager/bipfinder/publication_2.json")
.getPath();
@Test SparkAtomicActionScoreJob
public void matchOneWithTwo() throws Exception { .main(
String bipScoresPath = getClass().getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json").getPath(); new String[] {
String inputPath = getClass() "-isSparkSessionManaged",
.getResource( Boolean.FALSE.toString(),
"/eu/dnetlib/dhp/actionmanager/bipfinder/publication_2.json") "-inputPath",
.getPath(); inputPath,
"-bipScorePath",
bipScoresPath,
"-resultTableName",
"eu.dnetlib.dhp.schema.oaf.Publication",
"-outputPath",
workingDir.toString() + "/actionSet"
});
SparkAtomicActionScoreJob.main( final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
new String[] {
"-isSparkSessionManaged",
Boolean.FALSE.toString(),
"-inputPath",
inputPath,
"-bipScorePath",
bipScoresPath,
"-resultTableName",
"eu.dnetlib.dhp.schema.oaf.Publication",
"-outputPath",
workingDir.toString() + "/actionSet"
});
JavaRDD<Publication> tmp = sc
.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
.map(aa -> ((Publication) aa.getPayload()));
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); Assertions.assertTrue(tmp.count() == 1);
JavaRDD<Publication> tmp = sc Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class));
.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class) verificationDataset.createOrReplaceTempView("publication");
.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
.map(aa -> ((Publication) aa.getPayload()));
Assertions.assertTrue(tmp.count() == 1); Dataset<Row> execVerification = spark
.sql(
"Select p.id oaid, mes.id, mUnit.value from publication p " +
"lateral view explode(measures) m as mes " +
"lateral view explode(mes.unit) u as mUnit ");
Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class)); Assertions.assertEquals(4, execVerification.count());
verificationDataset.createOrReplaceTempView("publication");
Dataset<Row> execVerification = spark.sql("Select p.id oaid, mes.id, mUnit.value from publication p " + Assertions
"lateral view explode(measures) m as mes " + .assertEquals(
"lateral view explode(mes.unit) u as mUnit "); "50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb",
execVerification.select("oaid").collectAsList().get(0).getString(0));
Assertions.assertEquals(4, execVerification.count()); Assertions
.assertEquals(
2,
execVerification.filter("id = 'influence'").count());
Assertions
.assertEquals(
2,
execVerification.filter("id = 'popularity'").count());
Assertions.assertEquals("50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb", List<Row> tmp_ds = execVerification.filter("id = 'influence'").select("value").collectAsList();
execVerification.select("oaid").collectAsList().get(0).getString(0)); String tmp_influence = tmp_ds.get(0).getString(0);
Assertions
.assertTrue(
"1.47565045883e-08".equals(tmp_influence) ||
"1.98956540239e-08".equals(tmp_influence));
Assertions.assertEquals(2, tmp_influence = tmp_ds.get(1).getString(0);
execVerification.filter("id = 'influence'").count()); Assertions
.assertTrue(
"1.47565045883e-08".equals(tmp_influence) ||
"1.98956540239e-08".equals(tmp_influence));
Assertions.assertEquals(2, Assertions.assertTrue(!tmp_ds.get(0).getString(0).equals(tmp_ds.get(1).getString(0)));
execVerification.filter("id = 'popularity'").count());
List<Row> tmp_ds = execVerification.filter("id = 'influence'").select("value").collectAsList(); }
String tmp_influence = tmp_ds.get(0).getString(0);
Assertions.assertTrue("1.47565045883e-08".equals(tmp_influence)||
"1.98956540239e-08".equals(tmp_influence) );
tmp_influence = tmp_ds.get(1).getString(0); @Test
Assertions.assertTrue("1.47565045883e-08".equals(tmp_influence)|| public void matchTwo() throws Exception {
"1.98956540239e-08".equals(tmp_influence) ); String bipScoresPath = getClass()
.getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json")
.getPath();
String inputPath = getClass()
.getResource(
"/eu/dnetlib/dhp/actionmanager/bipfinder/publication_3.json")
.getPath();
Assertions.assertTrue(!tmp_ds.get(0).getString(0).equals(tmp_ds.get(1).getString(0))); SparkAtomicActionScoreJob
.main(
new String[] {
"-isSparkSessionManaged",
Boolean.FALSE.toString(),
"-inputPath",
inputPath,
"-bipScorePath",
bipScoresPath,
"-resultTableName",
"eu.dnetlib.dhp.schema.oaf.Publication",
"-outputPath",
workingDir.toString() + "/actionSet"
});
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
} JavaRDD<Publication> tmp = sc
.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
.map(aa -> ((Publication) aa.getPayload()));
@Test Assertions.assertTrue(tmp.count() == 2);
public void matchTwo() throws Exception {
String bipScoresPath = getClass().getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json").getPath();
String inputPath = getClass()
.getResource(
"/eu/dnetlib/dhp/actionmanager/bipfinder/publication_3.json")
.getPath();
SparkAtomicActionScoreJob.main( Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class));
new String[] { verificationDataset.createOrReplaceTempView("publication");
"-isSparkSessionManaged",
Boolean.FALSE.toString(),
"-inputPath",
inputPath,
"-bipScorePath",
bipScoresPath,
"-resultTableName",
"eu.dnetlib.dhp.schema.oaf.Publication",
"-outputPath",
workingDir.toString() + "/actionSet"
});
Dataset<Row> execVerification = spark
.sql(
"Select p.id oaid, mes.id, mUnit.value from publication p " +
"lateral view explode(measures) m as mes " +
"lateral view explode(mes.unit) u as mUnit ");
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); Assertions.assertEquals(4, execVerification.count());
JavaRDD<Publication> tmp = sc Assertions
.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class) .assertEquals(
.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) 2,
.map(aa -> ((Publication) aa.getPayload())); execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb'").count());
Assertions.assertTrue(tmp.count() == 2); Assertions
.assertEquals(
2,
execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09'").count());
Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class)); Assertions
verificationDataset.createOrReplaceTempView("publication"); .assertEquals(
2,
execVerification.filter("id = 'influence'").count());
Dataset<Row> execVerification = spark.sql("Select p.id oaid, mes.id, mUnit.value from publication p " + Assertions
"lateral view explode(measures) m as mes " + .assertEquals(
"lateral view explode(mes.unit) u as mUnit "); 2,
execVerification.filter("id = 'popularity'").count());
Assertions.assertEquals(4, execVerification.count()); Assertions
.assertEquals(
"1.47565045883e-08",
execVerification
.filter(
"oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " +
"and id = 'influence'")
.select("value")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"1.98956540239e-08",
execVerification
.filter(
"oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " +
"and id = 'influence'")
.select("value")
.collectAsList()
.get(0)
.getString(0));
Assertions.assertEquals(2, Assertions
execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb'").count()); .assertEquals(
"0.282046161584",
execVerification
.filter(
"oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " +
"and id = 'popularity'")
.select("value")
.collectAsList()
.get(0)
.getString(0));
Assertions.assertEquals(2, Assertions
execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09'").count()); .assertEquals(
"0.227515392",
execVerification
.filter(
"oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " +
"and id = 'popularity'")
.select("value")
.collectAsList()
.get(0)
.getString(0));
}
Assertions.assertEquals(2,
execVerification.filter("id = 'influence'").count());
Assertions.assertEquals(2,
execVerification.filter("id = 'popularity'").count());
Assertions.assertEquals("1.47565045883e-08" ,
execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " +
"and id = 'influence'").select("value").collectAsList().get(0).getString(0));
Assertions.assertEquals(
"1.98956540239e-08", execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " +
"and id = 'influence'").select("value").collectAsList().get(0).getString(0));
Assertions.assertEquals(
"0.282046161584", execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " +
"and id = 'popularity'").select("value").collectAsList().get(0).getString(0));
Assertions.assertEquals("0.227515392" ,
execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " +
"and id = 'popularity'").select("value").collectAsList().get(0).getString(0));
}
} }