From f0c476b6c957b354ba293ca3947887d53465fd2d Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 23 Sep 2020 17:31:49 +0200 Subject: [PATCH] modification to the test classes to consider h2020classification --- .../project/PrepareH2020ProgrammeTest.java | 65 ++++++++++++------- .../project/SparkUpdateProjectTest.java | 7 +- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java index 34e5c9729..4ba0891e7 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java @@ -5,12 +5,10 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.ForeachFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SparkSession; @@ -25,27 +23,27 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; -public class PrepareProgrammeTest { +public class PrepareH2020ProgrammeTest { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final ClassLoader cl = eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class + private static final ClassLoader cl = PrepareH2020ProgrammeTest.class .getClassLoader(); private static SparkSession spark; private static Path workingDir; private static final Logger log = LoggerFactory - .getLogger(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class); + .getLogger(PrepareH2020ProgrammeTest.class); @BeforeAll public static void beforeAll() throws IOException { workingDir = Files - .createTempDirectory(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class.getSimpleName()); + .createTempDirectory(PrepareH2020ProgrammeTest.class.getSimpleName()); log.info("using work dir {}", workingDir); SparkConf conf = new SparkConf(); - conf.setAppName(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class.getSimpleName()); + conf.setAppName(PrepareH2020ProgrammeTest.class.getSimpleName()); conf.setMaster("local[*]"); conf.set("spark.driver.host", "localhost"); @@ -56,7 +54,7 @@ public class PrepareProgrammeTest { spark = SparkSession .builder() - .appName(PrepareProgrammeTest.class.getSimpleName()) + .appName(PrepareH2020ProgrammeTest.class.getSimpleName()) .config(conf) .getOrCreate(); } @@ -94,24 +92,45 @@ public class PrepareProgrammeTest { Assertions.assertEquals(0, verificationDataset.filter("classification = ''").count()); - Assertions.assertEquals("Societal challenges $ Smart, Green And Integrated Transport $ CLEANSKY2 $ IADP Fast Rotorcraft", - verificationDataset.filter("code = 'H2020-EU.3.4.5.3.'").select("classification").collectAsList() - .get(0).getString(0)); + Assertions + .assertEquals( + "Societal challenges | Smart, Green And Integrated Transport | CLEANSKY2 | IADP Fast Rotorcraft", + verificationDataset + .filter("code = 'H2020-EU.3.4.5.3.'") + .select("classification") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Euratom | Indirect actions | European Fusion Development Agreement", + verificationDataset + .filter("code = 'H2020-Euratom-1.9.'") + .select("classification") + .collectAsList() + .get(0) + .getString(0)); - Assertions.assertEquals("Euratom $ Indirect actions $ European Fusion Development Agreement", - verificationDataset.filter("code = 'H2020-Euratom-1.9.'").select("classification").collectAsList() - .get(0).getString(0)); - - - Assertions.assertEquals("Industrial leadership $ Leadership in enabling and industrial technologies $ Advanced manufacturing and processing $ New sustainable business models", - verificationDataset.filter("code = 'H2020-EU.2.1.5.4.'").select("classification").collectAsList() - .get(0).getString(0)); - - Assertions.assertEquals("Excellent science $ Future and Emerging Technologies (FET) $ FET Open", - verificationDataset.filter("code = 'H2020-EU.1.2.1.'").select("classification").collectAsList() - .get(0).getString(0)); + Assertions + .assertEquals( + "Industrial leadership | Leadership in enabling and industrial technologies | Advanced manufacturing and processing | New sustainable business models", + verificationDataset + .filter("code = 'H2020-EU.2.1.5.4.'") + .select("classification") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Excellent science | Future and Emerging Technologies (FET) | FET Open", + verificationDataset + .filter("code = 'H2020-EU.1.2.1.'") + .select("classification") + .collectAsList() + .get(0) + .getString(0)); } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java index 718cd8ebe..9839c77c1 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -73,7 +73,8 @@ public class SparkUpdateProjectTest { Boolean.FALSE.toString(), "-programmePath", getClass() - .getResource("/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz") + .getResource( + "/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_classification_whole.json.gz") .getPath(), "-projectPath", getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json").getPath(), @@ -88,7 +89,9 @@ public class SparkUpdateProjectTest { .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) .map(aa -> ((Project) aa.getPayload())); - Assertions.assertEquals(14, tmp.count()); + Assertions.assertEquals(15, tmp.count()); + + tmp.foreach(value -> System.out.println(OBJECT_MAPPER.writeValueAsString(value))); } }