antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

added check to the test

This commit is contained in:
Miriam Baglioni 2020-10-01 15:40:28 +02:00
parent 6e5db85b32
commit 83ea746163
1 changed files with 39 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java
View File

@ -4,12 +4,16 @@ package eu.dnetlib.dhp.actionmanager.project;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.List;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
@ -78,6 +82,8 @@ public class SparkUpdateProjectTest {
.getPath(), .getPath(),
"-projectPath", "-projectPath",
getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json").getPath(), getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json").getPath(),
"-topicPath",
getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/topic.json.gz").getPath(),
"-outputPath", "-outputPath",
workingDir.toString() + "/actionSet" workingDir.toString() + "/actionSet"
}); });
@ -91,7 +97,39 @@ public class SparkUpdateProjectTest {
Assertions.assertEquals(15, tmp.count()); Assertions.assertEquals(15, tmp.count());
tmp.foreach(value -> System.out.println(OBJECT_MAPPER.writeValueAsString(value))); Dataset<Project> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Project.class));
verificationDataset.createOrReplaceTempView("project");
Dataset<Row> execverification = spark.sql("SELECT id, class classification, h2020topiccode, h2020topicdescription FROM project LATERAL VIEW EXPLODE(h2020classification) c as class ");
Assertions.assertEquals("H2020-EU.3.4.7.", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0));
Assertions.assertEquals("SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0));
Assertions.assertEquals("Societal challenges", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level1").collectAsList().get(0).getString(0));
Assertions.assertEquals("Smart, Green And Integrated Transport", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level2").collectAsList().get(0).getString(0));
Assertions.assertEquals("SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level3").collectAsList().get(0).getString(0));
Assertions.assertEquals("Societal challenges | Smart, Green And Integrated Transport | SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.classification").collectAsList().get(0).getString(0));
Assertions.assertEquals("SESAR-ER4-31-2019", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("h2020topiccode").collectAsList().get(0).getString(0));
Assertions.assertEquals("U-space", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("h2020topicdescription").collectAsList().get(0).getString(0));
Assertions.assertEquals("H2020-EU.1.3.2.", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0));
Assertions.assertEquals("Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0));
Assertions.assertEquals("Excellent science", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level1").collectAsList().get(0).getString(0));
Assertions.assertEquals("Marie Skłodowska-Curie Actions", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level2").collectAsList().get(0).getString(0));
Assertions.assertEquals("Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level3").collectAsList().get(0).getString(0));
Assertions.assertEquals("Excellent science | Marie Skłodowska-Curie Actions | Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.classification").collectAsList().get(0).getString(0));
Assertions.assertEquals("MSCA-IF-2019", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("h2020topiccode").collectAsList().get(0).getString(0));
Assertions.assertEquals("Individual Fellowships", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("h2020topicdescription").collectAsList().get(0).getString(0));
Assertions.assertTrue(execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0).equals("H2020-EU.2.1.4.") ||
execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(1).getString(0).equals("H2020-EU.2.1.4."));
Assertions.assertTrue(execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0).equals("H2020-EU.3.2.6.") ||
execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(1).getString(0).equals("H2020-EU.3.2.6."));
Assertions.assertEquals("Biotechnology", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.2.1.4.'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0));
Assertions.assertEquals("Bio-based Industries Joint Technology Initiative (BBI-JTI)", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.3.2.6.'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0));
Assertions.assertEquals("BBI-2019-SO3-D4", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("h2020topiccode").collectAsList().get(0).getString(0));
Assertions.assertEquals("Demonstrate bio-based pesticides and/or biostimulant agents for sustainable increase in agricultural productivity", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("h2020topicdescription").collectAsList().get(0).getString(0));
} }
} }