antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

modification to the test classes to consider h2020classification

This commit is contained in:
Miriam Baglioni 2020-09-23 17:31:49 +02:00
parent 2cba3cb484
commit f0c476b6c9
2 changed files with 47 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareH2020ProgrammeTest.java
View File

@ -5,12 +5,10 @@ import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.ForeachFunction;
import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
@ -25,27 +23,27 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme;
public class PrepareProgrammeTest { public class PrepareH2020ProgrammeTest {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final ClassLoader cl = eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class private static final ClassLoader cl = PrepareH2020ProgrammeTest.class
.getClassLoader(); .getClassLoader();
private static SparkSession spark; private static SparkSession spark;
private static Path workingDir; private static Path workingDir;
private static final Logger log = LoggerFactory private static final Logger log = LoggerFactory
.getLogger(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class); .getLogger(PrepareH2020ProgrammeTest.class);
@BeforeAll @BeforeAll
public static void beforeAll() throws IOException { public static void beforeAll() throws IOException {
workingDir = Files workingDir = Files
.createTempDirectory(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class.getSimpleName()); .createTempDirectory(PrepareH2020ProgrammeTest.class.getSimpleName());
log.info("using work dir {}", workingDir); log.info("using work dir {}", workingDir);
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();
conf.setAppName(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class.getSimpleName()); conf.setAppName(PrepareH2020ProgrammeTest.class.getSimpleName());
conf.setMaster("local[*]"); conf.setMaster("local[*]");
conf.set("spark.driver.host", "localhost"); conf.set("spark.driver.host", "localhost");
@ -56,7 +54,7 @@ public class PrepareProgrammeTest {
spark = SparkSession spark = SparkSession
.builder() .builder()
.appName(PrepareProgrammeTest.class.getSimpleName()) .appName(PrepareH2020ProgrammeTest.class.getSimpleName())
.config(conf) .config(conf)
.getOrCreate(); .getOrCreate();
} }
@ -94,24 +92,45 @@ public class PrepareProgrammeTest {
Assertions.assertEquals(0, verificationDataset.filter("classification = ''").count()); Assertions.assertEquals(0, verificationDataset.filter("classification = ''").count());
Assertions.assertEquals("Societal challenges $ Smart, Green And Integrated Transport $ CLEANSKY2 $ IADP Fast Rotorcraft", Assertions
verificationDataset.filter("code = 'H2020-EU.3.4.5.3.'").select("classification").collectAsList() .assertEquals(
.get(0).getString(0)); "Societal challenges | Smart, Green And Integrated Transport | CLEANSKY2 | IADP Fast Rotorcraft",
verificationDataset
.filter("code = 'H2020-EU.3.4.5.3.'")
.select("classification")
.collectAsList()
.get(0)
.getString(0));
Assertions
.assertEquals(
"Euratom | Indirect actions | European Fusion Development Agreement",
verificationDataset
.filter("code = 'H2020-Euratom-1.9.'")
.select("classification")
.collectAsList()
.get(0)
.getString(0));
Assertions.assertEquals("Euratom $ Indirect actions $ European Fusion Development Agreement", Assertions
verificationDataset.filter("code = 'H2020-Euratom-1.9.'").select("classification").collectAsList() .assertEquals(
.get(0).getString(0)); "Industrial leadership | Leadership in enabling and industrial technologies | Advanced manufacturing and processing | New sustainable business models",
verificationDataset
.filter("code = 'H2020-EU.2.1.5.4.'")
Assertions.assertEquals("Industrial leadership $ Leadership in enabling and industrial technologies $ Advanced manufacturing and processing $ New sustainable business models", .select("classification")
verificationDataset.filter("code = 'H2020-EU.2.1.5.4.'").select("classification").collectAsList() .collectAsList()
.get(0).getString(0)); .get(0)
.getString(0));
Assertions.assertEquals("Excellent science $ Future and Emerging Technologies (FET) $ FET Open",
verificationDataset.filter("code = 'H2020-EU.1.2.1.'").select("classification").collectAsList()
.get(0).getString(0));
Assertions
.assertEquals(
"Excellent science | Future and Emerging Technologies (FET) | FET Open",
verificationDataset
.filter("code = 'H2020-EU.1.2.1.'")
.select("classification")
.collectAsList()
.get(0)
.getString(0));
} }

7
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java
View File

@ -73,7 +73,8 @@ public class SparkUpdateProjectTest {
Boolean.FALSE.toString(), Boolean.FALSE.toString(),
"-programmePath", "-programmePath",
getClass() getClass()
.getResource("/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz") .getResource(
"/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_classification_whole.json.gz")
.getPath(), .getPath(),
"-projectPath", "-projectPath",
getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json").getPath(), getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json").getPath(),
@ -88,7 +89,9 @@ public class SparkUpdateProjectTest {
.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
.map(aa -> ((Project) aa.getPayload())); .map(aa -> ((Project) aa.getPayload()));
Assertions.assertEquals(14, tmp.count()); Assertions.assertEquals(15, tmp.count());
tmp.foreach(value -> System.out.println(OBJECT_MAPPER.writeValueAsString(value)));
} }
} }