2019-10-24 16:00:28 +02:00
|
|
|
package eu.dnetlib.dhp.graph;
|
|
|
|
|
2020-03-26 18:26:40 +01:00
|
|
|
import org.apache.spark.SparkConf;
|
2020-03-25 18:25:12 +01:00
|
|
|
import org.apache.spark.sql.SparkSession;
|
|
|
|
import org.junit.jupiter.api.Assertions;
|
|
|
|
import org.junit.jupiter.api.Test;
|
|
|
|
import org.junit.jupiter.api.io.TempDir;
|
2019-11-04 17:41:01 +01:00
|
|
|
|
|
|
|
import java.nio.file.Path;
|
2019-10-24 16:00:28 +02:00
|
|
|
|
|
|
|
public class SparkGraphImporterJobTest {
|
|
|
|
|
2020-03-26 18:26:40 +01:00
|
|
|
private final static String TEST_DB_NAME = "test";
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testImport(@TempDir Path outPath) {
|
|
|
|
try(SparkSession spark = testSparkSession(outPath.toString())) {
|
|
|
|
|
|
|
|
new SparkGraphImporterJob().runWith(
|
|
|
|
spark,
|
|
|
|
getClass().getResource("/eu/dnetlib/dhp/graph/sample").getPath(),
|
|
|
|
TEST_DB_NAME);
|
|
|
|
|
|
|
|
GraphMappingUtils.types.forEach((name, clazz) -> {
|
|
|
|
final long count = spark.read().table(TEST_DB_NAME + "." + name).count();
|
|
|
|
if (name.equals("relation")) {
|
|
|
|
Assertions.assertEquals(100, count, String.format("%s should be 100", name));
|
|
|
|
} else {
|
|
|
|
Assertions.assertEquals(10, count, String.format("%s should be 10", name));
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
2019-10-24 16:00:28 +02:00
|
|
|
}
|
2020-03-25 18:25:12 +01:00
|
|
|
|
2020-03-26 18:26:40 +01:00
|
|
|
private SparkSession testSparkSession(final String inputPath) {
|
|
|
|
SparkConf conf = new SparkConf();
|
2020-03-25 18:25:12 +01:00
|
|
|
|
2020-03-26 18:26:40 +01:00
|
|
|
conf.set("spark.driver.host", "localhost");
|
|
|
|
conf.set("hive.metastore.local", "true");
|
|
|
|
conf.set("hive.metastore.warehouse.dir", inputPath + "/warehouse");
|
|
|
|
conf.set("spark.sql.warehouse.dir", inputPath);
|
|
|
|
conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s/junit_metastore_db;create=true", inputPath));
|
|
|
|
conf.set("spark.ui.enabled", "false");
|
|
|
|
|
|
|
|
return SparkSession
|
2020-03-25 18:25:12 +01:00
|
|
|
.builder()
|
|
|
|
.appName(SparkGraphImporterJobTest.class.getSimpleName())
|
|
|
|
.master("local[*]")
|
2020-03-26 18:26:40 +01:00
|
|
|
.config(conf)
|
|
|
|
.enableHiveSupport()
|
2020-03-25 18:25:12 +01:00
|
|
|
.getOrCreate();
|
|
|
|
}
|
2020-03-26 18:26:40 +01:00
|
|
|
|
2019-10-24 16:00:28 +02:00
|
|
|
}
|