D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

Adding H2020 Classification, topic code and topic description to H2020 projects #46

Merged
claudio.atzori merged 59 commits from miriam.baglioni/dnet-hadoop:h2020classification into master 2020-10-05 14:14:39 +02:00
Conversation 1 Commits 59 Files Changed 35 +16 -18
1 changed files with 16 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit f8f5cfd5cc - Show all commits

34
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java
View File

@ -6,9 +6,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import java.util.*;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.sql.Dataset;
@ -20,8 +18,8 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme;
import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject;
import eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme;
import eu.dnetlib.dhp.actionmanager.project.utils.CSVProject;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.HdfsSupport;
import scala.Tuple2;
@ -97,25 +95,25 @@ public class PrepareProjects {
if (csvProject.isPresent()) {
String[] programme = csvProject.get().getProgramme().split(";");
String topic = csvProject.get().getTopics();
String topicdescription = Optional
.ofNullable(csvProject.get().getTopics())
.map(topics -> {
if (topic.equalsIgnoreCase(value._1().getTopiccode())) {
return value._1().getTopicdescription();
}
return null;
})
.orElse(null);
// String topic = csvProject.get().getTopics();
// String topicdescription = Optional
claudio.atzori commented 2020-10-01 17:37:22 +02:00
Outdated
Review
Copy Link

please remove unused code lines

please remove unused code lines
// .ofNullable(csvProject.get().getTopics())
// .map(topics -> {
// if (topic.equalsIgnoreCase(value._1().getTopiccode())) {
// return value._1().getTopicdescription();
// }
// return null;
// })
// .orElse(null);
Arrays
.stream(programme)
.forEach(p -> {
CSVProject proj = new CSVProject();
proj.setTopics(topic);
if (topicdescription != null) {
proj.setTopicdescription(topicdescription);
}
// proj.setTopics(topic);
// if (topicdescription != null) {
// proj.setTopicdescription(topicdescription);
// }
proj.setProgramme(p);
proj.setId(csvProject.get().getId());