From 83ea7461639987c4e3f75cdd3e83f2af28754013 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 1 Oct 2020 15:40:28 +0200 Subject: [PATCH] added check to the test --- .../project/SparkUpdateProjectTest.java | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java index 9839c77c1..f227bed11 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -4,12 +4,16 @@ package eu.dnetlib.dhp.actionmanager.project; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import org.apache.commons.io.FileUtils; import org.apache.hadoop.io.Text; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -78,6 +82,8 @@ public class SparkUpdateProjectTest { .getPath(), "-projectPath", getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json").getPath(), + "-topicPath", + getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/topic.json.gz").getPath(), "-outputPath", workingDir.toString() + "/actionSet" }); @@ -91,7 +97,39 @@ public class SparkUpdateProjectTest { Assertions.assertEquals(15, tmp.count()); - tmp.foreach(value -> System.out.println(OBJECT_MAPPER.writeValueAsString(value))); + Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Project.class)); + verificationDataset.createOrReplaceTempView("project"); + + Dataset execverification = spark.sql("SELECT id, class classification, h2020topiccode, h2020topicdescription FROM project LATERAL VIEW EXPLODE(h2020classification) c as class "); + + + Assertions.assertEquals("H2020-EU.3.4.7.", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0)); + Assertions.assertEquals("SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Societal challenges", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level1").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Smart, Green And Integrated Transport", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level2").collectAsList().get(0).getString(0)); + Assertions.assertEquals("SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level3").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Societal challenges | Smart, Green And Integrated Transport | SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.classification").collectAsList().get(0).getString(0)); + Assertions.assertEquals("SESAR-ER4-31-2019", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("h2020topiccode").collectAsList().get(0).getString(0)); + Assertions.assertEquals("U-space", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("h2020topicdescription").collectAsList().get(0).getString(0)); + + + Assertions.assertEquals("H2020-EU.1.3.2.", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Excellent science", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level1").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Marie Skłodowska-Curie Actions", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level2").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level3").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Excellent science | Marie Skłodowska-Curie Actions | Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.classification").collectAsList().get(0).getString(0)); + Assertions.assertEquals("MSCA-IF-2019", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("h2020topiccode").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Individual Fellowships", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("h2020topicdescription").collectAsList().get(0).getString(0)); + + Assertions.assertTrue(execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0).equals("H2020-EU.2.1.4.") || + execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(1).getString(0).equals("H2020-EU.2.1.4.")); + Assertions.assertTrue(execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0).equals("H2020-EU.3.2.6.") || + execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(1).getString(0).equals("H2020-EU.3.2.6.")); + Assertions.assertEquals("Biotechnology", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.2.1.4.'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Bio-based Industries Joint Technology Initiative (BBI-JTI)", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.3.2.6.'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); + Assertions.assertEquals("BBI-2019-SO3-D4", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("h2020topiccode").collectAsList().get(0).getString(0)); + Assertions.assertEquals("Demonstrate bio-based pesticides and/or biostimulant agents for sustainable increase in agricultural productivity", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("h2020topicdescription").collectAsList().get(0).getString(0)); } }