andreas.czerniak
/
BrBETA_dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

removed the part added to set the topic code and description in the step of project preparation

This commit is contained in:
Miriam Baglioni 2020-09-28 12:13:33 +02:00
parent 9e19c9a221
commit f8f5cfd5cc
1 changed files with 16 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java
View File

@ -6,9 +6,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import java.util.*; import java.util.*;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Dataset;
@ -20,8 +18,8 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; import eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme;
import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; import eu.dnetlib.dhp.actionmanager.project.utils.CSVProject;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.HdfsSupport; import eu.dnetlib.dhp.common.HdfsSupport;
import scala.Tuple2; import scala.Tuple2;
@ -97,25 +95,25 @@ public class PrepareProjects {
if (csvProject.isPresent()) { if (csvProject.isPresent()) {
String[] programme = csvProject.get().getProgramme().split(";"); String[] programme = csvProject.get().getProgramme().split(";");
String topic = csvProject.get().getTopics(); // String topic = csvProject.get().getTopics();
String topicdescription = Optional // String topicdescription = Optional
.ofNullable(csvProject.get().getTopics()) // .ofNullable(csvProject.get().getTopics())
.map(topics -> { // .map(topics -> {
if (topic.equalsIgnoreCase(value._1().getTopiccode())) { // if (topic.equalsIgnoreCase(value._1().getTopiccode())) {
return value._1().getTopicdescription(); // return value._1().getTopicdescription();
} // }
return null; // return null;
}) // })
.orElse(null); // .orElse(null);
Arrays Arrays
.stream(programme) .stream(programme)
.forEach(p -> { .forEach(p -> {
CSVProject proj = new CSVProject(); CSVProject proj = new CSVProject();
proj.setTopics(topic); // proj.setTopics(topic);
if (topicdescription != null) { // if (topicdescription != null) {
proj.setTopicdescription(topicdescription); // proj.setTopicdescription(topicdescription);
} // }
proj.setProgramme(p); proj.setProgramme(p);
proj.setId(csvProject.get().getId()); proj.setId(csvProject.get().getId());