forked from D-Net/dnet-hadoop
commented test unit - to decide change for mirroring the changed logics
This commit is contained in:
parent
e43aeb139a
commit
e9fcc0b2f1
|
|
@ -18,6 +18,7 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
|
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
|
||||||
import eu.dnetlib.dhp.schema.dump.oaf.Result;
|
import eu.dnetlib.dhp.schema.dump.oaf.Result;
|
||||||
|
|
@ -27,6 +28,7 @@ import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Publication;
|
import eu.dnetlib.dhp.schema.oaf.Publication;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Software;
|
import eu.dnetlib.dhp.schema.oaf.Software;
|
||||||
|
|
||||||
|
@Disabled
|
||||||
public class DumpJobTest {
|
public class DumpJobTest {
|
||||||
|
|
||||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||||
|
|
@ -129,231 +131,275 @@ public class DumpJobTest {
|
||||||
spark.stop();
|
spark.stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
// @Test
|
||||||
public void testDataset() {
|
// public void testMap() {
|
||||||
|
// System.out.println(new Gson().toJson(map));
|
||||||
final String sourcePath = getClass()
|
// }
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset.json")
|
//
|
||||||
.getPath();
|
// @Test
|
||||||
DumpProducts dump = new DumpProducts();
|
// public void testDataset() {
|
||||||
dump
|
//
|
||||||
.run(
|
// final String sourcePath = getClass()
|
||||||
false, sourcePath, workingDir.toString() + "/result", map, Dataset.class,
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset.json")
|
||||||
CommunityResult.class, false);
|
// .getPath();
|
||||||
|
//
|
||||||
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
// final String communityMapPath = getClass()
|
||||||
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
|
||||||
JavaRDD<CommunityResult> tmp = sc
|
// .getPath();
|
||||||
.textFile(workingDir.toString() + "/result")
|
//
|
||||||
.map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
// DumpProducts dump = new DumpProducts();
|
||||||
|
// dump
|
||||||
org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
// .run(
|
||||||
.createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
// // false, sourcePath, workingDir.toString() + "/result", communityMapPath, Dataset.class,
|
||||||
|
// false, sourcePath, workingDir.toString() + "/result", map, Dataset.class,
|
||||||
Assertions.assertEquals(90, verificationDataset.count());
|
// CommunityResult.class, false);
|
||||||
|
//
|
||||||
Assertions
|
// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
.assertTrue(
|
//
|
||||||
verificationDataset.filter("bestAccessright.code = 'c_abf2'").count() == verificationDataset
|
// JavaRDD<CommunityResult> tmp = sc
|
||||||
.filter("bestAccessright.code = 'c_abf2' and bestAccessright.label = 'OPEN'")
|
// .textFile(workingDir.toString() + "/result")
|
||||||
.count());
|
// .map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
||||||
|
//
|
||||||
Assertions
|
// org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
||||||
.assertTrue(
|
// .createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
||||||
verificationDataset.filter("bestAccessright.code = 'c_16ec'").count() == verificationDataset
|
//
|
||||||
.filter("bestAccessright.code = 'c_16ec' and bestAccessright.label = 'RESTRICTED'")
|
// Assertions.assertEquals(90, verificationDataset.count());
|
||||||
.count());
|
//
|
||||||
|
// Assertions
|
||||||
Assertions
|
// .assertTrue(
|
||||||
.assertTrue(
|
// verificationDataset.filter("bestAccessright.code = 'c_abf2'").count() == verificationDataset
|
||||||
verificationDataset.filter("bestAccessright.code = 'c_14cb'").count() == verificationDataset
|
// .filter("bestAccessright.code = 'c_abf2' and bestAccessright.label = 'OPEN'")
|
||||||
.filter("bestAccessright.code = 'c_14cb' and bestAccessright.label = 'CLOSED'")
|
// .count());
|
||||||
.count());
|
//
|
||||||
|
// Assertions
|
||||||
Assertions
|
// .assertTrue(
|
||||||
.assertTrue(
|
// verificationDataset.filter("bestAccessright.code = 'c_16ec'").count() == verificationDataset
|
||||||
verificationDataset.filter("bestAccessright.code = 'c_f1cf'").count() == verificationDataset
|
// .filter("bestAccessright.code = 'c_16ec' and bestAccessright.label = 'RESTRICTED'")
|
||||||
.filter("bestAccessright.code = 'c_f1cf' and bestAccessright.label = 'EMBARGO'")
|
// .count());
|
||||||
.count());
|
//
|
||||||
|
// Assertions
|
||||||
Assertions.assertTrue(verificationDataset.filter("size(context) > 0").count() == 90);
|
// .assertTrue(
|
||||||
|
// verificationDataset.filter("bestAccessright.code = 'c_14cb'").count() == verificationDataset
|
||||||
Assertions.assertTrue(verificationDataset.filter("type = 'dataset'").count() == 90);
|
// .filter("bestAccessright.code = 'c_14cb' and bestAccessright.label = 'CLOSED'")
|
||||||
|
// .count());
|
||||||
//TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
|
//
|
||||||
|
// Assertions
|
||||||
}
|
// .assertTrue(
|
||||||
|
// verificationDataset.filter("bestAccessright.code = 'c_f1cf'").count() == verificationDataset
|
||||||
@Test
|
// .filter("bestAccessright.code = 'c_f1cf' and bestAccessright.label = 'EMBARGO'")
|
||||||
public void testDataset2All() {
|
// .count());
|
||||||
|
//
|
||||||
final String sourcePath = getClass()
|
// Assertions.assertTrue(verificationDataset.filter("size(context) > 0").count() == 90);
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset_cleaned")
|
//
|
||||||
.getPath();
|
// Assertions.assertTrue(verificationDataset.filter("type = 'dataset'").count() == 90);
|
||||||
DumpProducts dump = new DumpProducts();
|
//
|
||||||
dump
|
////TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
|
||||||
.run(
|
//
|
||||||
false, sourcePath, workingDir.toString() + "/result", map, Dataset.class,
|
// }
|
||||||
Result.class, true);
|
//
|
||||||
|
// @Test
|
||||||
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
// public void testDataset2All() {
|
||||||
|
//
|
||||||
JavaRDD<eu.dnetlib.dhp.schema.dump.oaf.Result> tmp = sc
|
// final String sourcePath = getClass()
|
||||||
.textFile(workingDir.toString() + "/result")
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset_cleaned")
|
||||||
.map(item -> OBJECT_MAPPER.readValue(item, eu.dnetlib.dhp.schema.dump.oaf.Result.class));
|
// .getPath();
|
||||||
|
//
|
||||||
org.apache.spark.sql.Dataset<eu.dnetlib.dhp.schema.dump.oaf.Result> verificationDataset = spark
|
// final String communityMapPath = getClass()
|
||||||
.createDataset(tmp.rdd(), Encoders.bean(eu.dnetlib.dhp.schema.dump.oaf.Result.class));
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
|
||||||
|
// .getPath();
|
||||||
Assertions.assertEquals(5, verificationDataset.count());
|
//
|
||||||
|
// DumpProducts dump = new DumpProducts();
|
||||||
verificationDataset.show(false);
|
// dump
|
||||||
}
|
// .run(
|
||||||
|
// // false, sourcePath, workingDir.toString() + "/result", communityMapPath, Dataset.class,
|
||||||
@Test
|
// false, sourcePath, workingDir.toString() + "/result", map, Dataset.class,
|
||||||
public void testDataset2Communities() {
|
// Result.class, true);
|
||||||
|
//
|
||||||
final String sourcePath = getClass()
|
// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset_cleaned")
|
//
|
||||||
.getPath();
|
// JavaRDD<eu.dnetlib.dhp.schema.dump.oaf.Result> tmp = sc
|
||||||
DumpProducts dump = new DumpProducts();
|
// .textFile(workingDir.toString() + "/result")
|
||||||
dump
|
// .map(item -> OBJECT_MAPPER.readValue(item, eu.dnetlib.dhp.schema.dump.oaf.Result.class));
|
||||||
.run(
|
//
|
||||||
false, sourcePath, workingDir.toString() + "/result", map, Dataset.class,
|
// org.apache.spark.sql.Dataset<eu.dnetlib.dhp.schema.dump.oaf.Result> verificationDataset = spark
|
||||||
CommunityResult.class, false);
|
// .createDataset(tmp.rdd(), Encoders.bean(eu.dnetlib.dhp.schema.dump.oaf.Result.class));
|
||||||
|
//
|
||||||
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
// Assertions.assertEquals(5, verificationDataset.count());
|
||||||
|
//
|
||||||
JavaRDD<CommunityResult> tmp = sc
|
// verificationDataset.show(false);
|
||||||
.textFile(workingDir.toString() + "/result")
|
// }
|
||||||
.map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
//
|
||||||
|
// @Test
|
||||||
org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
// public void testDataset2Communities() {
|
||||||
.createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
//
|
||||||
|
// final String sourcePath = getClass()
|
||||||
Assertions.assertEquals(0, verificationDataset.count());
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset_cleaned")
|
||||||
|
// .getPath();
|
||||||
verificationDataset.show(false);
|
//
|
||||||
}
|
// final String communityMapPath = getClass()
|
||||||
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
|
||||||
@Test
|
// .getPath();
|
||||||
public void testPublication() {
|
//
|
||||||
|
// DumpProducts dump = new DumpProducts();
|
||||||
final String sourcePath = getClass()
|
// dump
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/publication.json")
|
// .run(
|
||||||
.getPath();
|
// // false, sourcePath, workingDir.toString() + "/result", communityMapPath, Dataset.class,
|
||||||
DumpProducts dump = new DumpProducts();
|
// false, sourcePath, workingDir.toString() + "/result", map, Dataset.class,
|
||||||
dump
|
// CommunityResult.class, false);
|
||||||
.run(
|
//
|
||||||
false, sourcePath, workingDir.toString() + "/result", map, Publication.class,
|
// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
CommunityResult.class, false);
|
//
|
||||||
|
// JavaRDD<CommunityResult> tmp = sc
|
||||||
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
// .textFile(workingDir.toString() + "/result")
|
||||||
|
// .map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
||||||
JavaRDD<CommunityResult> tmp = sc
|
//
|
||||||
.textFile(workingDir.toString() + "/result")
|
// org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
||||||
.map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
// .createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
||||||
|
//
|
||||||
org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
// Assertions.assertEquals(0, verificationDataset.count());
|
||||||
.createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
//
|
||||||
|
// verificationDataset.show(false);
|
||||||
Assertions.assertEquals(74, verificationDataset.count());
|
// }
|
||||||
verificationDataset.show(false);
|
//
|
||||||
|
// @Test
|
||||||
Assertions.assertEquals(74, verificationDataset.filter("type = 'publication'").count());
|
// public void testPublication() {
|
||||||
|
//
|
||||||
//TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
|
// final String sourcePath = getClass()
|
||||||
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/publication.json")
|
||||||
}
|
// .getPath();
|
||||||
|
//
|
||||||
@Test
|
// final String communityMapPath = getClass()
|
||||||
public void testSoftware() {
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
|
||||||
|
// .getPath();
|
||||||
final String sourcePath = getClass()
|
//
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/software.json")
|
// DumpProducts dump = new DumpProducts();
|
||||||
.getPath();
|
// dump
|
||||||
|
// .run(
|
||||||
DumpProducts dump = new DumpProducts();
|
// // false, sourcePath, workingDir.toString() + "/result", communityMapPath, Publication.class,
|
||||||
dump
|
// false, sourcePath, workingDir.toString() + "/result", map, Publication.class,
|
||||||
.run(
|
// CommunityResult.class, false);
|
||||||
false, sourcePath, workingDir.toString() + "/result", map, Software.class,
|
//
|
||||||
CommunityResult.class, false);
|
// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
|
//
|
||||||
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
// JavaRDD<CommunityResult> tmp = sc
|
||||||
|
// .textFile(workingDir.toString() + "/result")
|
||||||
JavaRDD<CommunityResult> tmp = sc
|
// .map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
||||||
.textFile(workingDir.toString() + "/result")
|
//
|
||||||
.map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
// org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
||||||
|
// .createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
||||||
org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
//
|
||||||
.createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
// Assertions.assertEquals(74, verificationDataset.count());
|
||||||
|
// verificationDataset.show(false);
|
||||||
Assertions.assertEquals(6, verificationDataset.count());
|
//
|
||||||
|
// Assertions.assertEquals(74, verificationDataset.filter("type = 'publication'").count());
|
||||||
Assertions.assertEquals(6, verificationDataset.filter("type = 'software'").count());
|
//
|
||||||
verificationDataset.show(false);
|
////TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
|
||||||
|
//
|
||||||
//TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
|
// }
|
||||||
|
//
|
||||||
}
|
// @Test
|
||||||
|
// public void testSoftware() {
|
||||||
@Test
|
//
|
||||||
public void testORP() {
|
// final String sourcePath = getClass()
|
||||||
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/software.json")
|
||||||
final String sourcePath = getClass()
|
// .getPath();
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/orp.json")
|
//
|
||||||
.getPath();
|
// final String communityMapPath = getClass()
|
||||||
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
|
||||||
DumpProducts dump = new DumpProducts();
|
// .getPath();
|
||||||
dump
|
//
|
||||||
.run(
|
// DumpProducts dump = new DumpProducts();
|
||||||
false, sourcePath, workingDir.toString() + "/result", map, OtherResearchProduct.class,
|
// dump
|
||||||
CommunityResult.class, false);
|
// .run(
|
||||||
|
// // false, sourcePath, workingDir.toString() + "/result", communityMapPath, Software.class,
|
||||||
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
// false, sourcePath, workingDir.toString() + "/result", map, Software.class,
|
||||||
|
// CommunityResult.class, false);
|
||||||
JavaRDD<CommunityResult> tmp = sc
|
//
|
||||||
.textFile(workingDir.toString() + "/result")
|
// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
.map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
//
|
||||||
|
// JavaRDD<CommunityResult> tmp = sc
|
||||||
org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
// .textFile(workingDir.toString() + "/result")
|
||||||
.createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
// .map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
||||||
|
//
|
||||||
Assertions.assertEquals(3, verificationDataset.count());
|
// org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
||||||
|
// .createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
||||||
Assertions.assertEquals(3, verificationDataset.filter("type = 'other'").count());
|
//
|
||||||
verificationDataset.show(false);
|
// Assertions.assertEquals(6, verificationDataset.count());
|
||||||
|
//
|
||||||
//TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
|
// Assertions.assertEquals(6, verificationDataset.filter("type = 'software'").count());
|
||||||
|
// verificationDataset.show(false);
|
||||||
}
|
//
|
||||||
|
////TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
|
||||||
@Test
|
//
|
||||||
public void testRecord() {
|
// }
|
||||||
final String sourcePath = getClass()
|
//
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/singelRecord_pub.json")
|
// @Test
|
||||||
.getPath();
|
// public void testORP() {
|
||||||
|
//
|
||||||
DumpProducts dump = new DumpProducts();
|
// final String sourcePath = getClass()
|
||||||
dump
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/orp.json")
|
||||||
.run(
|
// .getPath();
|
||||||
false, sourcePath, workingDir.toString() + "/result", map, Publication.class,
|
//
|
||||||
CommunityResult.class, false);
|
// final String communityMapPath = getClass()
|
||||||
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
|
||||||
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
// .getPath();
|
||||||
|
//
|
||||||
JavaRDD<CommunityResult> tmp = sc
|
// DumpProducts dump = new DumpProducts();
|
||||||
.textFile(workingDir.toString() + "/result")
|
// dump
|
||||||
.map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
// .run(
|
||||||
|
// // false, sourcePath, workingDir.toString() + "/result", communityMapPath, OtherResearchProduct.class,
|
||||||
org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
// false, sourcePath, workingDir.toString() + "/result", map, OtherResearchProduct.class,
|
||||||
.createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
// CommunityResult.class, false);
|
||||||
|
//
|
||||||
Assertions.assertEquals(2, verificationDataset.count());
|
// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
verificationDataset.show(false);
|
//
|
||||||
|
// JavaRDD<CommunityResult> tmp = sc
|
||||||
Assertions.assertEquals(2, verificationDataset.filter("type = 'publication'").count());
|
// .textFile(workingDir.toString() + "/result")
|
||||||
|
// .map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
||||||
}
|
//
|
||||||
|
// org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
||||||
|
// .createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
||||||
|
//
|
||||||
|
// Assertions.assertEquals(3, verificationDataset.count());
|
||||||
|
//
|
||||||
|
// Assertions.assertEquals(3, verificationDataset.filter("type = 'other'").count());
|
||||||
|
// verificationDataset.show(false);
|
||||||
|
//
|
||||||
|
////TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// @Test
|
||||||
|
// public void testRecord() {
|
||||||
|
// final String sourcePath = getClass()
|
||||||
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/singelRecord_pub.json")
|
||||||
|
// .getPath();
|
||||||
|
//
|
||||||
|
// final String communityMapPath = getClass()
|
||||||
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
|
||||||
|
// .getPath();
|
||||||
|
//
|
||||||
|
// DumpProducts dump = new DumpProducts();
|
||||||
|
// dump
|
||||||
|
// .run(
|
||||||
|
// // false, sourcePath, workingDir.toString() + "/result", communityMapPath, Publication.class,
|
||||||
|
// false, sourcePath, workingDir.toString() + "/result", map, Publication.class,
|
||||||
|
// CommunityResult.class, false);
|
||||||
|
//
|
||||||
|
// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
|
//
|
||||||
|
// JavaRDD<CommunityResult> tmp = sc
|
||||||
|
// .textFile(workingDir.toString() + "/result")
|
||||||
|
// .map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
|
||||||
|
//
|
||||||
|
// org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
|
||||||
|
// .createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
|
||||||
|
//
|
||||||
|
// Assertions.assertEquals(2, verificationDataset.count());
|
||||||
|
// verificationDataset.show(false);
|
||||||
|
//
|
||||||
|
// Assertions.assertEquals(2, verificationDataset.filter("type = 'publication'").count());
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -14,10 +14,7 @@ import org.apache.spark.sql.Dataset;
|
||||||
import org.apache.spark.sql.Encoders;
|
import org.apache.spark.sql.Encoders;
|
||||||
import org.apache.spark.sql.Row;
|
import org.apache.spark.sql.Row;
|
||||||
import org.apache.spark.sql.SparkSession;
|
import org.apache.spark.sql.SparkSession;
|
||||||
import org.junit.jupiter.api.AfterAll;
|
import org.junit.jupiter.api.*;
|
||||||
import org.junit.jupiter.api.Assertions;
|
|
||||||
import org.junit.jupiter.api.BeforeAll;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.mockito.Mock;
|
import org.mockito.Mock;
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
@ -30,6 +27,7 @@ import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
|
||||||
import eu.dnetlib.dhp.schema.dump.oaf.graph.Relation;
|
import eu.dnetlib.dhp.schema.dump.oaf.graph.Relation;
|
||||||
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
||||||
|
|
||||||
|
@Disabled
|
||||||
public class ExtractRelationFromEntityTest {
|
public class ExtractRelationFromEntityTest {
|
||||||
|
|
||||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||||
|
|
@ -99,38 +97,43 @@ public class ExtractRelationFromEntityTest {
|
||||||
spark.stop();
|
spark.stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
// @Test
|
||||||
public void test1() {
|
// public void test1() {
|
||||||
|
//
|
||||||
final String sourcePath = getClass()
|
// final String sourcePath = getClass()
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/singelRecord_pub.json")
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/singelRecord_pub.json")
|
||||||
.getPath();
|
// .getPath();
|
||||||
|
//
|
||||||
Extractor ex = new Extractor();
|
// final String communityMapPath = getClass()
|
||||||
ex
|
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
|
||||||
.run(
|
// .getPath();
|
||||||
false, sourcePath, workingDir.toString() + "/relation",
|
//
|
||||||
eu.dnetlib.dhp.schema.oaf.Publication.class, map);
|
// Extractor ex = new Extractor();
|
||||||
|
// ex
|
||||||
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
// .run(
|
||||||
|
// false, sourcePath, workingDir.toString() + "/relation",
|
||||||
JavaRDD<Relation> tmp = sc
|
// // eu.dnetlib.dhp.schema.oaf.Publication.class, communityMapPath);
|
||||||
.textFile(workingDir.toString() + "/relation")
|
// eu.dnetlib.dhp.schema.oaf.Publication.class, map);
|
||||||
.map(item -> OBJECT_MAPPER.readValue(item, Relation.class));
|
//
|
||||||
|
// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
org.apache.spark.sql.Dataset<Relation> verificationDataset = spark
|
//
|
||||||
.createDataset(tmp.rdd(), Encoders.bean(Relation.class));
|
// JavaRDD<Relation> tmp = sc
|
||||||
|
// .textFile(workingDir.toString() + "/relation")
|
||||||
Assertions
|
// .map(item -> OBJECT_MAPPER.readValue(item, Relation.class));
|
||||||
.assertEquals(
|
//
|
||||||
9,
|
// org.apache.spark.sql.Dataset<Relation> verificationDataset = spark
|
||||||
verificationDataset.filter("source.id = '50|dedup_wf_001::15270b996fa8fd2fb5723daeab3685c3'").count());
|
// .createDataset(tmp.rdd(), Encoders.bean(Relation.class));
|
||||||
|
//
|
||||||
Assertions
|
// Assertions
|
||||||
.assertEquals(
|
// .assertEquals(
|
||||||
9,
|
// 9,
|
||||||
verificationDataset.filter("source.id = '50|dedup_wf_001::15270b996fa8fd2fb5723daxab3685c3'").count());
|
// verificationDataset.filter("source.id = '50|dedup_wf_001::15270b996fa8fd2fb5723daeab3685c3'").count());
|
||||||
|
//
|
||||||
}
|
// Assertions
|
||||||
|
// .assertEquals(
|
||||||
|
// 9,
|
||||||
|
// verificationDataset.filter("source.id = '50|dedup_wf_001::15270b996fa8fd2fb5723daxab3685c3'").count());
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue