pispis
/
dnet-hadoop
forked from antonis.lempesis/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

removed the part added to set the topic code and description in the step of project preparation

This commit is contained in:
Miriam Baglioni 2020-09-28 12:13:33 +02:00
parent 9e19c9a221
commit f8f5cfd5cc
1 changed files with 16 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java
View File

@ -6,9 +6,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import java.util.*;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.sql.Dataset;
@ -20,8 +18,8 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme;
import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject;
import eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme;
import eu.dnetlib.dhp.actionmanager.project.utils.CSVProject;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.HdfsSupport;
import scala.Tuple2;
@ -97,25 +95,25 @@ public class PrepareProjects {
if (csvProject.isPresent()) {
String[] programme = csvProject.get().getProgramme().split(";");
String topic = csvProject.get().getTopics();
String topicdescription = Optional
.ofNullable(csvProject.get().getTopics())
.map(topics -> {
if (topic.equalsIgnoreCase(value._1().getTopiccode())) {
return value._1().getTopicdescription();
}
return null;
})
.orElse(null);
// String topic = csvProject.get().getTopics();
// String topicdescription = Optional
// .ofNullable(csvProject.get().getTopics())
// .map(topics -> {
// if (topic.equalsIgnoreCase(value._1().getTopiccode())) {
// return value._1().getTopicdescription();
// }
// return null;
// })
// .orElse(null);
Arrays
.stream(programme)
.forEach(p -> {
CSVProject proj = new CSVProject();
proj.setTopics(topic);
if (topicdescription != null) {
proj.setTopicdescription(topicdescription);
}
// proj.setTopics(topic);
// if (topicdescription != null) {
// proj.setTopicdescription(topicdescription);
// }
proj.setProgramme(p);
proj.setId(csvProject.get().getId());