sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

test for project propagation

This commit is contained in:
Miriam Baglioni 2023-10-31 15:49:42 +01:00
parent e9a20fc8f6
commit c80b768af0
3 changed files with 36 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java
View File

@ -56,7 +56,7 @@ public class PrepareResultCommunitySet {
log.info("production: {}", production); log.info("production: {}", production);
final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production); final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production);
log.info("projectsMap: {}", new Gson().toJson(projectsMap)); //log.info("projectsMap: {}", new Gson().toJson(projectsMap));
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();

14
dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java
View File

@ -10,9 +10,7 @@ import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.*;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -61,6 +59,7 @@ public class PrepareAssocTest {
spark.stop(); spark.stop();
} }
@Disabled
@Test @Test
void test1() throws Exception { void test1() throws Exception {
@ -72,8 +71,8 @@ public class PrepareAssocTest {
"-sourcePath", "-sourcePath",
getClass().getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/relation/").getPath(), getClass().getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/relation/").getPath(),
"-outputPath", workingDir.toString() + "/prepared", "-outputPath", workingDir.toString() + "/prepared",
"-production", Boolean.TRUE.toString(), "-production", Boolean.TRUE.toString()
"-hive_metastore_uris", ""
}); });
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
@ -82,7 +81,10 @@ public class PrepareAssocTest {
.textFile(workingDir.toString() + "/prepared") .textFile(workingDir.toString() + "/prepared")
.map(item -> new ObjectMapper().readValue(item, ResultProjectList.class)); .map(item -> new ObjectMapper().readValue(item, ResultProjectList.class));
tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); Assertions.assertEquals(4, tmp.count());
Assertions.assertEquals(2, tmp.filter(rpl -> rpl.getCommunityList().contains("aurora")).count());
Assertions.assertEquals(1, tmp.filter(rpl -> rpl.getCommunityList().contains("sdsn-gr")).count());
Assertions.assertEquals(1, tmp.filter(rpl -> rpl.getCommunityList().contains("netherlands")).count());
} }
} }

257
dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java
View File

@ -6,7 +6,9 @@ import static org.apache.spark.sql.functions.desc;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.List;
import eu.dnetlib.dhp.schema.oaf.Context;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
@ -89,235 +91,30 @@ public class ResultToCommunityJobTest {
.textFile(workingDir.toString() + "/dataset") .textFile(workingDir.toString() + "/dataset")
.map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class));
tmp.foreach(d -> System.out.println(new ObjectMapper().writeValueAsString(d))); Assertions.assertEquals(10, tmp.count());
// Assertions.assertEquals(10, tmp.count()); /**
// org.apache.spark.sql.Dataset<Dataset> verificationDataset = spark * {"resultId":"50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f","communityList":["aurora"]}
// .createDataset(tmp.rdd(), Encoders.bean(Dataset.class)); * {"resultId":"50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e","communityList":["aurora"]}
// * {"resultId":"50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1","communityList":["sdsn-gr"]}
// verificationDataset.createOrReplaceTempView("dataset"); * {"resultId":"50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1","communityList":["netherlands"]}
// */
// String query = "select id, MyT.id community " List<Context> context = tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f"))
// + "from dataset " .first().getContext();
// + "lateral view explode(context) c as MyT " Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c)));
// + "lateral view explode(MyT.datainfo) d as MyD "
// + "where MyD.inferenceprovenance = 'propagation'"; context = tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e"))
// .first().getContext();
// org.apache.spark.sql.Dataset<Row> resultExplodedProvenance = spark.sql(query); Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c)));
// Assertions.assertEquals(5, resultExplodedProvenance.count());
// Assertions Assertions.assertEquals(0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1")).count());
// .assertEquals(
// 0, Assertions.assertEquals(0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1")).count());
// resultExplodedProvenance
// .filter("id = '50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe'") Assertions.assertEquals(2, tmp.filter(r -> r.getContext().stream().anyMatch(c -> c.getId().equals("aurora"))).count());
// .count());
// Assertions }
// .assertEquals(
// 1, private static boolean containsResultCommunityProject(Context c) {
// resultExplodedProvenance return c.getDataInfo().stream().anyMatch(di -> di.getProvenanceaction().getClassid().equals("result:community:project"));
// .filter("id = '50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b'")
// .count());
// Assertions
// .assertEquals(
// "beopen",
// resultExplodedProvenance
// .select("community")
// .where(
// resultExplodedProvenance
// .col("id")
// .equalTo(
// "50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b"))
// .collectAsList()
// .get(0)
// .getString(0));
//
// Assertions
// .assertEquals(
// 2,
// resultExplodedProvenance
// .filter("id = '50|od________18::8887b1df8b563c4ea851eb9c882c9d7b'")
// .count());
// Assertions
// .assertEquals(
// "mes",
// resultExplodedProvenance
// .select("community")
// .where(
// resultExplodedProvenance
// .col("id")
// .equalTo(
// "50|od________18::8887b1df8b563c4ea851eb9c882c9d7b"))
// .sort(desc("community"))
// .collectAsList()
// .get(0)
// .getString(0));
// Assertions
// .assertEquals(
// "euromarine",
// resultExplodedProvenance
// .select("community")
// .where(
// resultExplodedProvenance
// .col("id")
// .equalTo(
// "50|od________18::8887b1df8b563c4ea851eb9c882c9d7b"))
// .sort(desc("community"))
// .collectAsList()
// .get(1)
// .getString(0));
//
// Assertions
// .assertEquals(
// 1,
// resultExplodedProvenance
// .filter("id = '50|doajarticles::8d817039a63710fcf97e30f14662c6c8'")
// .count());
// Assertions
// .assertEquals(
// "mes",
// resultExplodedProvenance
// .select("community")
// .where(
// resultExplodedProvenance
// .col("id")
// .equalTo(
// "50|doajarticles::8d817039a63710fcf97e30f14662c6c8"))
// .sort(desc("community"))
// .collectAsList()
// .get(0)
// .getString(0));
//
// Assertions
// .assertEquals(
// 1,
// resultExplodedProvenance
// .filter("id = '50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6'")
// .count());
// Assertions
// .assertEquals(
// "mes",
// resultExplodedProvenance
// .select("community")
// .where(
// resultExplodedProvenance
// .col("id")
// .equalTo(
// "50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6"))
// .sort(desc("community"))
// .collectAsList()
// .get(0)
// .getString(0));
//
// query = "select id, MyT.id community "
// + "from dataset "
// + "lateral view explode(context) c as MyT "
// + "lateral view explode(MyT.datainfo) d as MyD ";
//
// org.apache.spark.sql.Dataset<Row> resultCommunityId = spark.sql(query);
//
// Assertions.assertEquals(10, resultCommunityId.count());
//
// Assertions
// .assertEquals(
// 1,
// resultCommunityId
// .filter("id = '50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe'")
// .count());
// Assertions
// .assertEquals(
// "beopen",
// resultCommunityId
// .select("community")
// .where(
// resultCommunityId
// .col("id")
// .equalTo(
// "50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe"))
// .collectAsList()
// .get(0)
// .getString(0));
//
// Assertions
// .assertEquals(
// 1,
// resultCommunityId
// .filter("id = '50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b'")
// .count());
//
// Assertions
// .assertEquals(
// 3,
// resultCommunityId
// .filter("id = '50|od________18::8887b1df8b563c4ea851eb9c882c9d7b'")
// .count());
// Assertions
// .assertEquals(
// "beopen",
// resultCommunityId
// .select("community")
// .where(
// resultCommunityId
// .col("id")
// .equalTo(
// "50|od________18::8887b1df8b563c4ea851eb9c882c9d7b"))
// .sort(desc("community"))
// .collectAsList()
// .get(2)
// .getString(0));
//
// Assertions
// .assertEquals(
// 2,
// resultCommunityId
// .filter("id = '50|doajarticles::8d817039a63710fcf97e30f14662c6c8'")
// .count());
// Assertions
// .assertEquals(
// "euromarine",
// resultCommunityId
// .select("community")
// .where(
// resultCommunityId
// .col("id")
// .equalTo(
// "50|doajarticles::8d817039a63710fcf97e30f14662c6c8"))
// .sort(desc("community"))
// .collectAsList()
// .get(1)
// .getString(0));
//
// Assertions
// .assertEquals(
// 3,
// resultCommunityId
// .filter("id = '50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6'")
// .count());
// Assertions
// .assertEquals(
// "euromarine",
// resultCommunityId
// .select("community")
// .where(
// resultCommunityId
// .col("id")
// .equalTo(
// "50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6"))
// .sort(desc("community"))
// .collectAsList()
// .get(2)
// .getString(0));
// Assertions
// .assertEquals(
// "ni",
// resultCommunityId
// .select("community")
// .where(
// resultCommunityId
// .col("id")
// .equalTo(
// "50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6"))
// .sort(desc("community"))
// .collectAsList()
// .get(0)
// .getString(0));
} }
} }