From 61946b409227ebb1cb76b35b58e0eff09ec79e1f Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 1 Oct 2020 16:22:48 +0200 Subject: [PATCH] refactoring --- .../project/SparkAtomicActionJob.java | 3 +- .../actionmanager/project/CSVParserTest.java | 2 - .../project/EXCELParserTest.java | 1 - .../project/PrepareH2020ProgrammeTest.java | 16 +- .../project/SparkUpdateProjectTest.java | 244 ++++++++++++++++-- 5 files changed, 226 insertions(+), 40 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java index 555bd17574..6bf784bbe9 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java @@ -108,7 +108,6 @@ public class SparkAtomicActionJob { CSVProject csvProject = c._1(); Optional ocsvProgramme = Optional.ofNullable(c._2()); - return Optional .ofNullable(c._2()) .map(csvProgramme -> { @@ -172,7 +171,7 @@ public class SparkAtomicActionJob { if (tmp.length > 2) { h2020Classification.setLevel3(tmp[2]); } - h2020Classification.getH2020Programme().setDescription(tmp[tmp.length-1]); + h2020Classification.getH2020Programme().setDescription(tmp[tmp.length - 1]); } public static Dataset readPath( diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java index 3ec98262ae..da5beecf9a 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java @@ -1,7 +1,6 @@ package eu.dnetlib.dhp.actionmanager.project; - import java.util.List; import org.apache.commons.io.IOUtils; @@ -12,7 +11,6 @@ import eu.dnetlib.dhp.actionmanager.project.utils.CSVParser; public class CSVParserTest { - @Test public void readProgrammeTest() throws Exception { diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/EXCELParserTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/EXCELParserTest.java index c64459472d..fdc577687c 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/EXCELParserTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/EXCELParserTest.java @@ -38,6 +38,5 @@ public class EXCELParserTest { Assertions.assertEquals(3837, pl.size()); - } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java index 2ecea6f0b7..c5801bcccb 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java @@ -133,14 +133,14 @@ public class PrepareH2020ProgrammeTest { .getString(0)); Assertions - .assertEquals( - "Industrial leadership | Leadership in enabling and industrial technologies | Biotechnology", - verificationDataset - .filter("code = 'H2020-EU.2.1.4.'") - .select("classification") - .collectAsList() - .get(0) - .getString(0)); + .assertEquals( + "Industrial leadership | Leadership in enabling and industrial technologies | Biotechnology", + verificationDataset + .filter("code = 'H2020-EU.2.1.4.'") + .select("classification") + .collectAsList() + .get(0) + .getString(0)); } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java index f227bed113..cfda7e7181 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -100,36 +100,226 @@ public class SparkUpdateProjectTest { Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Project.class)); verificationDataset.createOrReplaceTempView("project"); - Dataset execverification = spark.sql("SELECT id, class classification, h2020topiccode, h2020topicdescription FROM project LATERAL VIEW EXPLODE(h2020classification) c as class "); + Dataset execverification = spark + .sql( + "SELECT id, class classification, h2020topiccode, h2020topicdescription FROM project LATERAL VIEW EXPLODE(h2020classification) c as class "); + Assertions + .assertEquals( + "H2020-EU.3.4.7.", + execverification + .filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'") + .select("classification.h2020Programme.code") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "SESAR JU", + execverification + .filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'") + .select("classification.h2020Programme.description") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Societal challenges", + execverification + .filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'") + .select("classification.level1") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Smart, Green And Integrated Transport", + execverification + .filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'") + .select("classification.level2") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "SESAR JU", + execverification + .filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'") + .select("classification.level3") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Societal challenges | Smart, Green And Integrated Transport | SESAR JU", + execverification + .filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'") + .select("classification.classification") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "SESAR-ER4-31-2019", + execverification + .filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'") + .select("h2020topiccode") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "U-space", + execverification + .filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'") + .select("h2020topicdescription") + .collectAsList() + .get(0) + .getString(0)); - Assertions.assertEquals("H2020-EU.3.4.7.", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0)); - Assertions.assertEquals("SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Societal challenges", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level1").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Smart, Green And Integrated Transport", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level2").collectAsList().get(0).getString(0)); - Assertions.assertEquals("SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level3").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Societal challenges | Smart, Green And Integrated Transport | SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.classification").collectAsList().get(0).getString(0)); - Assertions.assertEquals("SESAR-ER4-31-2019", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("h2020topiccode").collectAsList().get(0).getString(0)); - Assertions.assertEquals("U-space", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("h2020topicdescription").collectAsList().get(0).getString(0)); + Assertions + .assertEquals( + "H2020-EU.1.3.2.", + execverification + .filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'") + .select("classification.h2020Programme.code") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Nurturing excellence by means of cross-border and cross-sector mobility", + execverification + .filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'") + .select("classification.h2020Programme.description") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Excellent science", + execverification + .filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'") + .select("classification.level1") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Marie Skłodowska-Curie Actions", + execverification + .filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'") + .select("classification.level2") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Nurturing excellence by means of cross-border and cross-sector mobility", + execverification + .filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'") + .select("classification.level3") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Excellent science | Marie Skłodowska-Curie Actions | Nurturing excellence by means of cross-border and cross-sector mobility", + execverification + .filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'") + .select("classification.classification") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "MSCA-IF-2019", + execverification + .filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'") + .select("h2020topiccode") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Individual Fellowships", + execverification + .filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'") + .select("h2020topicdescription") + .collectAsList() + .get(0) + .getString(0)); - - Assertions.assertEquals("H2020-EU.1.3.2.", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Excellent science", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level1").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Marie Skłodowska-Curie Actions", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level2").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level3").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Excellent science | Marie Skłodowska-Curie Actions | Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.classification").collectAsList().get(0).getString(0)); - Assertions.assertEquals("MSCA-IF-2019", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("h2020topiccode").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Individual Fellowships", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("h2020topicdescription").collectAsList().get(0).getString(0)); - - Assertions.assertTrue(execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0).equals("H2020-EU.2.1.4.") || - execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(1).getString(0).equals("H2020-EU.2.1.4.")); - Assertions.assertTrue(execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0).equals("H2020-EU.3.2.6.") || - execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(1).getString(0).equals("H2020-EU.3.2.6.")); - Assertions.assertEquals("Biotechnology", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.2.1.4.'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Bio-based Industries Joint Technology Initiative (BBI-JTI)", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.3.2.6.'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); - Assertions.assertEquals("BBI-2019-SO3-D4", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("h2020topiccode").collectAsList().get(0).getString(0)); - Assertions.assertEquals("Demonstrate bio-based pesticides and/or biostimulant agents for sustainable increase in agricultural productivity", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("h2020topicdescription").collectAsList().get(0).getString(0)); + Assertions + .assertTrue( + execverification + .filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'") + .select("classification.h2020Programme.code") + .collectAsList() + .get(0) + .getString(0) + .equals("H2020-EU.2.1.4.") || + execverification + .filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'") + .select("classification.h2020Programme.code") + .collectAsList() + .get(1) + .getString(0) + .equals("H2020-EU.2.1.4.")); + Assertions + .assertTrue( + execverification + .filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'") + .select("classification.h2020Programme.code") + .collectAsList() + .get(0) + .getString(0) + .equals("H2020-EU.3.2.6.") || + execverification + .filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'") + .select("classification.h2020Programme.code") + .collectAsList() + .get(1) + .getString(0) + .equals("H2020-EU.3.2.6.")); + Assertions + .assertEquals( + "Biotechnology", + execverification + .filter( + "id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.2.1.4.'") + .select("classification.h2020Programme.description") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Bio-based Industries Joint Technology Initiative (BBI-JTI)", + execverification + .filter( + "id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.3.2.6.'") + .select("classification.h2020Programme.description") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "BBI-2019-SO3-D4", + execverification + .filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'") + .select("h2020topiccode") + .collectAsList() + .get(0) + .getString(0)); + Assertions + .assertEquals( + "Demonstrate bio-based pesticides and/or biostimulant agents for sustainable increase in agricultural productivity", + execverification + .filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'") + .select("h2020topicdescription") + .collectAsList() + .get(0) + .getString(0)); } }