D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 9 Projects 2 Releases 1 Wiki Activity

[Clean Context] changed a bit the logic. Added the check not to have result hosted by a datasource of type institutional repository from NL. Added also the check that the country should have been included in the result via propagation for it to be removed

This commit is contained in:
Miriam Baglioni 2022-08-08 14:10:47 +02:00
parent 390013a4b2
commit 62d2138806
5 changed files with 315 additions and 270 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/country/CleanCountrySparkJob.java
View File

@ -1,15 +1,15 @@
package eu.dnetlib.dhp.oa.graph.clean.country; package eu.dnetlib.dhp.oa.graph.clean.country;
/** import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
* @author miriam.baglioni
* @Date 20/07/22 import java.io.Serializable;
*/ import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.swing.text.html.Option;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.clean.CleanContextSparkJob;
import eu.dnetlib.dhp.schema.oaf.Result;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.FilterFunction;
@ -21,13 +21,17 @@ import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.Serializable; /**
import java.util.List; * @author miriam.baglioni
import java.util.Optional; * @Date 20/07/22
import java.util.stream.Collectors; */
import com.fasterxml.jackson.databind.ObjectMapper;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.clean.CleanContextSparkJob;
import eu.dnetlib.dhp.schema.oaf.Country;
import eu.dnetlib.dhp.schema.oaf.Result;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
public class CleanCountrySparkJob implements Serializable { public class CleanCountrySparkJob implements Serializable {
private static final Logger log = LoggerFactory.getLogger(CleanCountrySparkJob.class); private static final Logger log = LoggerFactory.getLogger(CleanCountrySparkJob.class);
@ -56,7 +60,7 @@ public class CleanCountrySparkJob implements Serializable {
String workingPath = parser.get("workingPath"); String workingPath = parser.get("workingPath");
log.info("workingPath: {}", workingPath); log.info("workingPath: {}", workingPath);
String datasourcePath = parser.get("datasourcePath"); String datasourcePath = parser.get("hostedBy");
log.info("datasourcePath: {}", datasourcePath); log.info("datasourcePath: {}", datasourcePath);
String country = parser.get("country"); String country = parser.get("country");
@ -79,14 +83,17 @@ public class CleanCountrySparkJob implements Serializable {
isSparkSessionManaged, isSparkSessionManaged,
spark -> { spark -> {
cleanCountry(spark, country, verifyParam, inputPath, entityClazz, workingPath,collectedfrom, datasourcePath); cleanCountry(
spark, country, verifyParam, inputPath, entityClazz, workingPath, collectedfrom, datasourcePath);
}); });
} }
private static <T extends Result> void cleanCountry(SparkSession spark, String country, String[] verifyParam, private static <T extends Result> void cleanCountry(SparkSession spark, String country, String[] verifyParam,
String inputPath, Class<T> entityClazz, String workingPath, String collectedfrom, String datasourcePath) { String inputPath, Class<T> entityClazz, String workingPath, String collectedfrom, String datasourcePath) {
List<String> hostedBy = spark.read().textFile(datasourcePath) List<String> hostedBy = spark
.read()
.textFile(datasourcePath)
// .filter((FilterFunction<String>) ds -> !ds.equals(collectedfrom)) // .filter((FilterFunction<String>) ds -> !ds.equals(collectedfrom))
.collectAsList(); .collectAsList();
@ -95,25 +102,25 @@ public class CleanCountrySparkJob implements Serializable {
.textFile(inputPath) .textFile(inputPath)
.map( .map(
(MapFunction<String, T>) value -> OBJECT_MAPPER.readValue(value, entityClazz), (MapFunction<String, T>) value -> OBJECT_MAPPER.readValue(value, entityClazz),
Encoders.bean(entityClazz)) Encoders.bean(entityClazz));
;
res.map((MapFunction<T, T>) r -> { res.map((MapFunction<T, T>) r -> {
if(r.getInstance().stream().anyMatch(i -> hostedBy.contains(i.getHostedby().getKey())) || if (r.getInstance().stream().anyMatch(i -> hostedBy.contains(i.getHostedby().getKey())) ||
!r.getCollectedfrom().stream().anyMatch(cf -> cf.getValue().equals(collectedfrom))){ !r.getCollectedfrom().stream().anyMatch(cf -> cf.getValue().equals(collectedfrom))) {
return r; return r;
} }
if(r.getPid().stream().anyMatch(p -> p.getQualifier().getClassid().equals("doi") && pidInParam(p.getValue(), verifyParam)) if (r
&& r.getCountry().stream().anyMatch(c -> c.getClassid().equals(country) && c.getDataInfo().getInferenceprovenance().equals("propagation"))) .getPid()
{ r .stream()
.anyMatch(p -> p.getQualifier().getClassid().equals("doi") && pidInParam(p.getValue(), verifyParam))) {
r
.setCountry( .setCountry(
r r
.getCountry() .getCountry()
.stream() .stream()
.filter( .filter(
c -> !c.getClassid() c -> toTakeCountry(c, country))
.equalsIgnoreCase(country))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -138,11 +145,23 @@ public class CleanCountrySparkJob implements Serializable {
} }
private static boolean pidInParam(String value, String[] verifyParam) { private static boolean pidInParam(String value, String[] verifyParam) {
for (String s : verifyParam ) for (String s : verifyParam)
if (value.startsWith(s)) if (value.startsWith(s))
return true; return true;
return false; return false;
} }
} private static boolean toTakeCountry(Country c, String country) {
// If dataInfo is not set, or dataInfo.inferenceprovenance is not set or not present then it cannot be
// inserted via propagation
if (!Optional.ofNullable(c.getDataInfo()).isPresent())
return true;
if (!Optional.ofNullable(c.getDataInfo().getInferenceprovenance()).isPresent())
return true;
return !(c
.getClassid()
.equalsIgnoreCase(country) &&
c.getDataInfo().getInferenceprovenance().equals("propagation"));
}
}

47
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/country/GetDatasourceFromCountry.java
View File

@ -1,10 +1,12 @@
package eu.dnetlib.dhp.oa.graph.clean.country; package eu.dnetlib.dhp.oa.graph.clean.country;
import com.fasterxml.jackson.databind.ObjectMapper; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.clean.CleanContextSparkJob; import java.io.Serializable;
import eu.dnetlib.dhp.schema.common.ModelConstants; import java.util.List;
import eu.dnetlib.dhp.schema.oaf.*; import java.util.Optional;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.FilterFunction;
@ -15,14 +17,15 @@ import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.clean.CleanContextSparkJob;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.*;
import scala.Tuple2; import scala.Tuple2;
import java.io.Serializable;
import java.util.List;
import java.util.Optional;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
/** /**
* @author miriam.baglioni * @author miriam.baglioni
* @Date 22/07/22 * @Date 22/07/22
@ -67,20 +70,25 @@ public class GetDatasourceFromCountry implements Serializable {
}); });
} }
private static void getDatasourceFromCountry(SparkSession spark, String country, String inputPath, String workingPath) { private static void getDatasourceFromCountry(SparkSession spark, String country, String inputPath,
String workingPath) {
Dataset<Organization> organization = spark.read().textFile(inputPath + "/organization") Dataset<Organization> organization = spark
.read()
.textFile(inputPath + "/organization")
.map( .map(
(MapFunction<String, Organization>) value -> OBJECT_MAPPER.readValue(value, Organization.class), (MapFunction<String, Organization>) value -> OBJECT_MAPPER.readValue(value, Organization.class),
Encoders.bean(Organization.class)) Encoders.bean(Organization.class))
.filter( .filter(
(FilterFunction<Organization>) o -> !o.getDataInfo().getDeletedbyinference() && (FilterFunction<Organization>) o -> !o.getDataInfo().getDeletedbyinference() &&
o.getCountry().getClassid().length() > 0 && o.getCountry().getClassid().length() > 0 &&
o.getCountry().getClassid().equals(country));; o.getCountry().getClassid().equals(country));
;
// filtering of the relations taking the non deleted by inference and those with IsProvidedBy as relclass // filtering of the relations taking the non deleted by inference and those with IsProvidedBy as relclass
Dataset<Relation> relation = spark.read().textFile( inputPath + "/relation") Dataset<Relation> relation = spark
.read()
.textFile(inputPath + "/relation")
.map( .map(
(MapFunction<String, Relation>) value -> OBJECT_MAPPER.readValue(value, Relation.class), (MapFunction<String, Relation>) value -> OBJECT_MAPPER.readValue(value, Relation.class),
Encoders.bean(Relation.class)) Encoders.bean(Relation.class))
@ -88,11 +96,12 @@ public class GetDatasourceFromCountry implements Serializable {
(FilterFunction<Relation>) rel -> rel.getRelClass().equalsIgnoreCase(ModelConstants.IS_PROVIDED_BY) && (FilterFunction<Relation>) rel -> rel.getRelClass().equalsIgnoreCase(ModelConstants.IS_PROVIDED_BY) &&
!rel.getDataInfo().getDeletedbyinference()); !rel.getDataInfo().getDeletedbyinference());
organization.joinWith(relation, organization.col("id").equalTo(relation.col("target")), "left") organization
.map((MapFunction<Tuple2<Organization, Relation>, String>) t2 -> t2._2().getSource(), Encoders.STRING() ) .joinWith(relation, organization.col("id").equalTo(relation.col("target")), "left")
.map((MapFunction<Tuple2<Organization, Relation>, String>) t2 -> t2._2().getSource(), Encoders.STRING())
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression","gzip") .option("compression", "gzip")
.json(workingPath); .json(workingPath);
} }

56
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleanCountryTest.java
View File

@ -1,13 +1,10 @@
package eu.dnetlib.dhp.oa.graph.clean;
/**
* @author miriam.baglioni
* @Date 20/07/22
*/
import com.fasterxml.jackson.databind.ObjectMapper; package eu.dnetlib.dhp.oa.graph.clean;
import eu.dnetlib.dhp.oa.graph.clean.country.CleanCountrySparkJob;
import eu.dnetlib.dhp.oa.graph.dump.DumpJobTest; import java.io.IOException;
import eu.dnetlib.dhp.schema.oaf.Publication; import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
@ -22,10 +19,14 @@ import org.junit.jupiter.api.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.IOException; /**
import java.nio.file.Files; * @author miriam.baglioni
import java.nio.file.Path; * @Date 20/07/22
*/
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.oa.graph.clean.country.CleanCountrySparkJob;
import eu.dnetlib.dhp.schema.oaf.Publication;
public class CleanCountryTest { public class CleanCountryTest {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@ -38,11 +39,11 @@ public class CleanCountryTest {
@BeforeAll @BeforeAll
public static void beforeAll() throws IOException { public static void beforeAll() throws IOException {
workingDir = Files.createTempDirectory(DumpJobTest.class.getSimpleName()); workingDir = Files.createTempDirectory(CleanCountryTest.class.getSimpleName());
log.info("using work dir {}", workingDir); log.info("using work dir {}", workingDir);
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();
conf.setAppName(DumpJobTest.class.getSimpleName()); conf.setAppName(CleanCountryTest.class.getSimpleName());
conf.setMaster("local[*]"); conf.setMaster("local[*]");
conf.set("spark.driver.host", "localhost"); conf.set("spark.driver.host", "localhost");
@ -53,7 +54,7 @@ public class CleanCountryTest {
spark = SparkSession spark = SparkSession
.builder() .builder()
.appName(DumpJobTest.class.getSimpleName()) .appName(CleanCountryTest.class.getSimpleName())
.config(conf) .config(conf)
.getOrCreate(); .getOrCreate();
} }
@ -69,7 +70,6 @@ public class CleanCountryTest {
final String sourcePath = getClass() final String sourcePath = getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/clean/publication_clean_country.json") .getResource("/eu/dnetlib/dhp/oa/graph/clean/publication_clean_country.json")
.getPath(); .getPath();
final String prefix = "gcube ";
spark spark
.read() .read()
@ -87,7 +87,10 @@ public class CleanCountryTest {
"-workingPath", workingDir.toString() + "/working", "-workingPath", workingDir.toString() + "/working",
"-country", "NL", "-country", "NL",
"-verifyParam", "10.17632", "-verifyParam", "10.17632",
"-collectedfrom", "NARCIS" "-collectedfrom", "NARCIS",
"-hostedBy", getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/clean/hostedBy")
.getPath()
}); });
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
@ -95,7 +98,7 @@ public class CleanCountryTest {
.textFile(workingDir.toString() + "/publication") .textFile(workingDir.toString() + "/publication")
.map(item -> OBJECT_MAPPER.readValue(item, Publication.class)); .map(item -> OBJECT_MAPPER.readValue(item, Publication.class));
Assertions.assertEquals(7, tmp.count()); Assertions.assertEquals(8, tmp.count());
// original result with NL country and doi starting with Mendely prefix, but not collectedfrom NARCIS // original result with NL country and doi starting with Mendely prefix, but not collectedfrom NARCIS
Assertions Assertions
@ -119,12 +122,25 @@ public class CleanCountryTest {
.getCountry() .getCountry()
.size()); .size());
// original result with NL country and doi starting with Mendely prefix and collectedfrom NARCIS // original result with NL country and doi starting with Mendely prefix and collectedfrom NARCIS but not
// inserted with propagation
Assertions
.assertEquals(
1,
tmp
.filter(p -> p.getId().equals("50|DansKnawCris::3c81248c335f0aa07e06817ece6fa6af"))
.collect()
.get(0)
.getCountry()
.size());
// original result with NL country and doi starting with Mendely prefix and collectedfrom NARCIS inserted with
// propagation
Assertions Assertions
.assertEquals( .assertEquals(
0, 0,
tmp tmp
.filter(p -> p.getId().equals("50|DansKnawCris::3c81248c335f0aa07e06817ece6fa6af")) .filter(p -> p.getId().equals("50|DansKnawCris::3c81248c335f0aa07e06817ece6fa6ag"))
.collect() .collect()
.get(0) .get(0)
.getCountry() .getCountry()

0
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/hostedBy Normal file
View File

1
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/publication_clean_country.json
View File

File diff suppressed because one or more lines are too long