dimitris.pierrakos
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

[Clean Country]changes related to D-Net/dnet-hadoop#241 (comment)

This commit is contained in:
Miriam Baglioni 2022-08-10 15:13:10 +02:00
parent 62d2138806
commit 7dbdd4a0fe
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/country/CleanCountrySparkJob.java
View File

@ -10,6 +10,7 @@ import java.util.stream.Collectors;
import javax.swing.text.html.Option;
import eu.dnetlib.dhp.schema.oaf.utils.PidType;
import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FilterFunction;
@ -94,7 +95,6 @@ public class CleanCountrySparkJob implements Serializable {
List<String> hostedBy = spark
.read()
.textFile(datasourcePath)
// .filter((FilterFunction<String>) ds -> !ds.equals(collectedfrom))
.collectAsList();
Dataset<T> res = spark
@ -113,7 +113,8 @@ public class CleanCountrySparkJob implements Serializable {
if (r
.getPid()
.stream()
.anyMatch(p -> p.getQualifier().getClassid().equals("doi") && pidInParam(p.getValue(), verifyParam))) {
.anyMatch(p -> p.getQualifier().getClassid()
.equals(PidType.doi) && pidInParam(p.getValue(), verifyParam))) {
r
.setCountry(
r