D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

refactoring

This commit is contained in:
Miriam Baglioni 2020-10-01 16:22:48 +02:00
parent 7e6d35e56c
commit 61946b4092
5 changed files with 226 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java
View File

@ -108,7 +108,6 @@ public class SparkAtomicActionJob {
CSVProject csvProject = c._1(); CSVProject csvProject = c._1();
Optional<CSVProgramme> ocsvProgramme = Optional.ofNullable(c._2()); Optional<CSVProgramme> ocsvProgramme = Optional.ofNullable(c._2());
return Optional return Optional
.ofNullable(c._2()) .ofNullable(c._2())
.map(csvProgramme -> { .map(csvProgramme -> {
@ -172,7 +171,7 @@ public class SparkAtomicActionJob {
if (tmp.length > 2) { if (tmp.length > 2) {
h2020Classification.setLevel3(tmp[2]); h2020Classification.setLevel3(tmp[2]);
} }
h2020Classification.getH2020Programme().setDescription(tmp[tmp.length-1]); h2020Classification.getH2020Programme().setDescription(tmp[tmp.length - 1]);
} }
public static <R> Dataset<R> readPath( public static <R> Dataset<R> readPath(

2
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.dhp.actionmanager.project; package eu.dnetlib.dhp.actionmanager.project;
import java.util.List; import java.util.List;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
@ -12,7 +11,6 @@ import eu.dnetlib.dhp.actionmanager.project.utils.CSVParser;
public class CSVParserTest { public class CSVParserTest {
@Test @Test
public void readProgrammeTest() throws Exception { public void readProgrammeTest() throws Exception {

1
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/EXCELParserTest.java
View File

@ -38,6 +38,5 @@ public class EXCELParserTest {
Assertions.assertEquals(3837, pl.size()); Assertions.assertEquals(3837, pl.size());
} }
} }

244
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java
View File

@ -100,36 +100,226 @@ public class SparkUpdateProjectTest {
Dataset<Project> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Project.class)); Dataset<Project> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Project.class));
verificationDataset.createOrReplaceTempView("project"); verificationDataset.createOrReplaceTempView("project");
Dataset<Row> execverification = spark.sql("SELECT id, class classification, h2020topiccode, h2020topicdescription FROM project LATERAL VIEW EXPLODE(h2020classification) c as class "); Dataset<Row> execverification = spark
.sql(
"SELECT id, class classification, h2020topiccode, h2020topicdescription FROM project LATERAL VIEW EXPLODE(h2020classification) c as class ");
Assertions
.assertEquals(
"H2020-EU.3.4.7.",
execverification
.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'")
.select("classification.h2020Programme.code")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"SESAR JU",
execverification
.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'")
.select("classification.h2020Programme.description")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Societal challenges",
execverification
.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'")
.select("classification.level1")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Smart, Green And Integrated Transport",
execverification
.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'")
.select("classification.level2")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"SESAR JU",
execverification
.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'")
.select("classification.level3")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Societal challenges | Smart, Green And Integrated Transport | SESAR JU",
execverification
.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'")
.select("classification.classification")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"SESAR-ER4-31-2019",
execverification
.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'")
.select("h2020topiccode")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"U-space",
execverification
.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'")
.select("h2020topicdescription")
.collectAsList()
.get(0)
.getString(0));
Assertions.assertEquals("H2020-EU.3.4.7.", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0)); Assertions
Assertions.assertEquals("SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); .assertEquals(
Assertions.assertEquals("Societal challenges", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level1").collectAsList().get(0).getString(0)); "H2020-EU.1.3.2.",
Assertions.assertEquals("Smart, Green And Integrated Transport", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level2").collectAsList().get(0).getString(0)); execverification
Assertions.assertEquals("SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.level3").collectAsList().get(0).getString(0)); .filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'")
Assertions.assertEquals("Societal challenges | Smart, Green And Integrated Transport | SESAR JU", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("classification.classification").collectAsList().get(0).getString(0)); .select("classification.h2020Programme.code")
Assertions.assertEquals("SESAR-ER4-31-2019", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("h2020topiccode").collectAsList().get(0).getString(0)); .collectAsList()
Assertions.assertEquals("U-space", execverification.filter("id = '40|corda__h2020::2c7298913008865ba784e5c1350a0aa5'").select("h2020topicdescription").collectAsList().get(0).getString(0)); .get(0)
.getString(0));
Assertions
.assertEquals(
"Nurturing excellence by means of cross-border and cross-sector mobility",
execverification
.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'")
.select("classification.h2020Programme.description")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Excellent science",
execverification
.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'")
.select("classification.level1")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Marie Skłodowska-Curie Actions",
execverification
.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'")
.select("classification.level2")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Nurturing excellence by means of cross-border and cross-sector mobility",
execverification
.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'")
.select("classification.level3")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Excellent science | Marie Skłodowska-Curie Actions | Nurturing excellence by means of cross-border and cross-sector mobility",
execverification
.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'")
.select("classification.classification")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"MSCA-IF-2019",
execverification
.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'")
.select("h2020topiccode")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Individual Fellowships",
execverification
.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'")
.select("h2020topicdescription")
.collectAsList()
.get(0)
.getString(0));
Assertions
Assertions.assertEquals("H2020-EU.1.3.2.", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0)); .assertTrue(
Assertions.assertEquals("Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); execverification
Assertions.assertEquals("Excellent science", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level1").collectAsList().get(0).getString(0)); .filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'")
Assertions.assertEquals("Marie Skłodowska-Curie Actions", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level2").collectAsList().get(0).getString(0)); .select("classification.h2020Programme.code")
Assertions.assertEquals("Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.level3").collectAsList().get(0).getString(0)); .collectAsList()
Assertions.assertEquals("Excellent science | Marie Skłodowska-Curie Actions | Nurturing excellence by means of cross-border and cross-sector mobility", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("classification.classification").collectAsList().get(0).getString(0)); .get(0)
Assertions.assertEquals("MSCA-IF-2019", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("h2020topiccode").collectAsList().get(0).getString(0)); .getString(0)
Assertions.assertEquals("Individual Fellowships", execverification.filter("id = '40|corda__h2020::1a1f235fdd06ef14790baec159aa1202'").select("h2020topicdescription").collectAsList().get(0).getString(0)); .equals("H2020-EU.2.1.4.") ||
execverification
Assertions.assertTrue(execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0).equals("H2020-EU.2.1.4.") || .filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'")
execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(1).getString(0).equals("H2020-EU.2.1.4.")); .select("classification.h2020Programme.code")
Assertions.assertTrue(execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(0).getString(0).equals("H2020-EU.3.2.6.") || .collectAsList()
execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("classification.h2020Programme.code").collectAsList().get(1).getString(0).equals("H2020-EU.3.2.6.")); .get(1)
Assertions.assertEquals("Biotechnology", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.2.1.4.'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); .getString(0)
Assertions.assertEquals("Bio-based Industries Joint Technology Initiative (BBI-JTI)", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.3.2.6.'").select("classification.h2020Programme.description").collectAsList().get(0).getString(0)); .equals("H2020-EU.2.1.4."));
Assertions.assertEquals("BBI-2019-SO3-D4", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("h2020topiccode").collectAsList().get(0).getString(0)); Assertions
Assertions.assertEquals("Demonstrate bio-based pesticides and/or biostimulant agents for sustainable increase in agricultural productivity", execverification.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'").select("h2020topicdescription").collectAsList().get(0).getString(0)); .assertTrue(
execverification
.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'")
.select("classification.h2020Programme.code")
.collectAsList()
.get(0)
.getString(0)
.equals("H2020-EU.3.2.6.") ||
execverification
.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'")
.select("classification.h2020Programme.code")
.collectAsList()
.get(1)
.getString(0)
.equals("H2020-EU.3.2.6."));
Assertions
.assertEquals(
"Biotechnology",
execverification
.filter(
"id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.2.1.4.'")
.select("classification.h2020Programme.description")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Bio-based Industries Joint Technology Initiative (BBI-JTI)",
execverification
.filter(
"id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5' and classification.h2020Programme.code = 'H2020-EU.3.2.6.'")
.select("classification.h2020Programme.description")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"BBI-2019-SO3-D4",
execverification
.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'")
.select("h2020topiccode")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Demonstrate bio-based pesticides and/or biostimulant agents for sustainable increase in agricultural productivity",
execverification
.filter("id = '40|corda__h2020::a657c271769fec90b60c1f2dbc25f4d5'")
.select("h2020topicdescription")
.collectAsList()
.get(0)
.getString(0));
} }
} }