andreas.czerniak
/
BrBETA_dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

modification to the test classes to consider h2020classification

This commit is contained in:
Miriam Baglioni 2020-09-23 17:31:49 +02:00
parent 2cba3cb484
commit f0c476b6c9
2 changed files with 47 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java
View File

@ -5,12 +5,10 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject;
import org.apache.commons.io.FileUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.ForeachFunction;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SparkSession;
@ -25,27 +23,27 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme;
public class PrepareProgrammeTest {
public class PrepareH2020ProgrammeTest {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final ClassLoader cl = eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class
private static final ClassLoader cl = PrepareH2020ProgrammeTest.class
.getClassLoader();
private static SparkSession spark;
private static Path workingDir;
private static final Logger log = LoggerFactory
.getLogger(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class);
.getLogger(PrepareH2020ProgrammeTest.class);
@BeforeAll
public static void beforeAll() throws IOException {
workingDir = Files
.createTempDirectory(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class.getSimpleName());
.createTempDirectory(PrepareH2020ProgrammeTest.class.getSimpleName());
log.info("using work dir {}", workingDir);
SparkConf conf = new SparkConf();
conf.setAppName(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class.getSimpleName());
conf.setAppName(PrepareH2020ProgrammeTest.class.getSimpleName());
conf.setMaster("local[*]");
conf.set("spark.driver.host", "localhost");
@ -56,7 +54,7 @@ public class PrepareProgrammeTest {
spark = SparkSession
.builder()
.appName(PrepareProgrammeTest.class.getSimpleName())
.appName(PrepareH2020ProgrammeTest.class.getSimpleName())
.config(conf)
.getOrCreate();
}
@ -94,24 +92,45 @@ public class PrepareProgrammeTest {
Assertions.assertEquals(0, verificationDataset.filter("classification = ''").count());
Assertions.assertEquals("Societal challenges $ Smart, Green And Integrated Transport $ CLEANSKY2 $ IADP Fast Rotorcraft",
verificationDataset.filter("code = 'H2020-EU.3.4.5.3.'").select("classification").collectAsList()
.get(0).getString(0));
Assertions
.assertEquals(
"Societal challenges | Smart, Green And Integrated Transport | CLEANSKY2 | IADP Fast Rotorcraft",
verificationDataset
.filter("code = 'H2020-EU.3.4.5.3.'")
.select("classification")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Euratom | Indirect actions | European Fusion Development Agreement",
verificationDataset
.filter("code = 'H2020-Euratom-1.9.'")
.select("classification")
.collectAsList()
.get(0)
.getString(0));
Assertions.assertEquals("Euratom $ Indirect actions $ European Fusion Development Agreement",
verificationDataset.filter("code = 'H2020-Euratom-1.9.'").select("classification").collectAsList()
.get(0).getString(0));
Assertions.assertEquals("Industrial leadership $ Leadership in enabling and industrial technologies $ Advanced manufacturing and processing $ New sustainable business models",
verificationDataset.filter("code = 'H2020-EU.2.1.5.4.'").select("classification").collectAsList()
.get(0).getString(0));
Assertions.assertEquals("Excellent science $ Future and Emerging Technologies (FET) $ FET Open",
verificationDataset.filter("code = 'H2020-EU.1.2.1.'").select("classification").collectAsList()
.get(0).getString(0));
Assertions
.assertEquals(
"Industrial leadership | Leadership in enabling and industrial technologies | Advanced manufacturing and processing | New sustainable business models",
verificationDataset
.filter("code = 'H2020-EU.2.1.5.4.'")
.select("classification")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Excellent science | Future and Emerging Technologies (FET) | FET Open",
verificationDataset
.filter("code = 'H2020-EU.1.2.1.'")
.select("classification")
.collectAsList()
.get(0)
.getString(0));
}

7
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java
View File

@ -73,7 +73,8 @@ public class SparkUpdateProjectTest {
Boolean.FALSE.toString(),
"-programmePath",
getClass()
.getResource("/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz")
.getResource(
"/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_classification_whole.json.gz")
.getPath(),
"-projectPath",
getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json").getPath(),
@ -88,7 +89,9 @@ public class SparkUpdateProjectTest {
.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
.map(aa -> ((Project) aa.getPayload()));
Assertions.assertEquals(14, tmp.count());
Assertions.assertEquals(15, tmp.count());
tmp.foreach(value -> System.out.println(OBJECT_MAPPER.writeValueAsString(value)));
}
}