antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

-

This commit is contained in:
Miriam Baglioni 2021-11-10 17:00:37 +01:00
parent 9e214ce0eb
commit c371b23077
23 changed files with 1319 additions and 847 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java
View File

@ -30,11 +30,11 @@ public class PropagationConstant {
public static final String UPDATE_DATA_INFO_TYPE = "update"; public static final String UPDATE_DATA_INFO_TYPE = "update";
public static final String UPDATE_SUBJECT_FOS_CLASS_ID = "subject:fos"; public static final String UPDATE_SUBJECT_FOS_CLASS_ID = "subject:fos";
public static final String UPDATE_SUBJECT_FOS_CLASS_NAME = "Update of results with FOS subjects"; public static final String UPDATE_CLASS_NAME = "Inferred by OpenAIRE";
public static final String UPDATE_MEASURE_BIP_CLASS_ID = "measure:bip"; public static final String UPDATE_MEASURE_BIP_CLASS_ID = "measure:bip";
public static final String UPDATE_MEASURE_BIP_CLASS_NAME = "Update of results with BipFinder! measures";
public static final String FOS_CLASS_ID = "FOS"; public static final String FOS_CLASS_ID = "FOS";
public static final String FOS_CLASS_NAME = "Subject from fos classification"; public static final String FOS_CLASS_NAME = "Fields of Science and Technology classification";
public static final String OPENCITATIONS_CLASSID = "sysimport:crosswalk:opencitations"; public static final String OPENCITATIONS_CLASSID = "sysimport:crosswalk:opencitations";
public static final String OPENCITATIONS_CLASSNAME = "Imported from OpenCitations"; public static final String OPENCITATIONS_CLASSNAME = "Imported from OpenCitations";
@ -98,15 +98,16 @@ public class PropagationConstant {
} }
public static DataInfo getDataInfo( public static DataInfo getDataInfo(
String inference_provenance, String inference_class_id, String inference_class_name, String qualifierSchema, String inference_provenance, String inference_class_id, String inference_class_name, String qualifierSchema,
String trust) { String trust) {
return getDataInfo(inference_provenance, inference_class_id, inference_class_name, qualifierSchema, trust, true); return getDataInfo(
inference_provenance, inference_class_id, inference_class_name, qualifierSchema, trust, true);
} }
public static DataInfo getDataInfo( public static DataInfo getDataInfo(
String inference_provenance, String inference_class_id, String inference_class_name, String qualifierSchema, String inference_provenance, String inference_class_id, String inference_class_name, String qualifierSchema,
String trust, boolean inferred) { String trust, boolean inferred) {
DataInfo di = new DataInfo(); DataInfo di = new DataInfo();
di.setInferred(inferred); di.setInferred(inferred);
di.setDeletedbyinference(false); di.setDeletedbyinference(false);

16
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bypassactionset/Utils.java
View File

@ -1,15 +1,17 @@
package eu.dnetlib.dhp.bypassactionset; package eu.dnetlib.dhp.bypassactionset;
import org.jetbrains.annotations.NotNull;
import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
import org.jetbrains.annotations.NotNull;
public class Utils { public class Utils {
private static final String ID_PREFIX = "50|doi_________"; private static final String ID_PREFIX = "50|doi_________";
@NotNull @NotNull
public static String getIdentifier(String d) { public static String getIdentifier(String d) {
return ID_PREFIX + return ID_PREFIX +
IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", d)); IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", d));
} }
} }

113
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bypassactionset/bipfinder/PrepareBipFinder.java
View File

@ -1,10 +1,13 @@
package eu.dnetlib.dhp.bypassactionset.bipfinder; package eu.dnetlib.dhp.bypassactionset.bipfinder;
import com.fasterxml.jackson.databind.ObjectMapper; import static eu.dnetlib.dhp.bypassactionset.Utils.getIdentifier;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import eu.dnetlib.dhp.bypassactionset.model.BipDeserialize;
import eu.dnetlib.dhp.bypassactionset.model.BipScore; import java.io.Serializable;
import eu.dnetlib.dhp.schema.oaf.Result; import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
@ -15,74 +18,72 @@ import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.Serializable; import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.Optional;
import java.util.stream.Collectors;
import static eu.dnetlib.dhp.bypassactionset.Utils.getIdentifier; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import eu.dnetlib.dhp.bypassactionset.model.BipDeserialize;
import eu.dnetlib.dhp.bypassactionset.model.BipScore;
import eu.dnetlib.dhp.schema.oaf.Result;
public class PrepareBipFinder implements Serializable { public class PrepareBipFinder implements Serializable {
private static final String DOI = "doi";
private static final Logger log = LoggerFactory.getLogger(SparkUpdateBip.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
public static <I extends Result> void main(String[] args) throws Exception { private static final Logger log = LoggerFactory.getLogger(SparkUpdateBip.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
String jsonConfiguration = IOUtils public static <I extends Result> void main(String[] args) throws Exception {
.toString(
SparkUpdateBip.class
.getResourceAsStream(
"/eu/dnetlib/dhp/actionmanager/bipfinder/input_parameters.json"));
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); String jsonConfiguration = IOUtils
.toString(
SparkUpdateBip.class
.getResourceAsStream(
"/eu/dnetlib/dhp/bypassactionset/bip_prepare_parameters.json"));
parser.parseArgument(args); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
Boolean isSparkSessionManaged = Optional parser.parseArgument(args);
.ofNullable(parser.get("isSparkSessionManaged"))
.map(Boolean::valueOf)
.orElse(Boolean.TRUE);
log.info("isSparkSessionManaged: {}", isSparkSessionManaged); Boolean isSparkSessionManaged = Optional
.ofNullable(parser.get("isSparkSessionManaged"))
.map(Boolean::valueOf)
.orElse(Boolean.TRUE);
final String inputPath = parser.get("inputPath"); log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
log.info("inputPath {}: ", inputPath);
final String outputPath = parser.get("outputPath"); final String inputPath = parser.get("inputPath");
log.info("outputPath {}: ", outputPath); log.info("inputPath {}: ", inputPath);
final String outputPath = parser.get("outputPath");
log.info("outputPath {}: ", outputPath);
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();
runWithSparkSession( runWithSparkSession(
conf, conf,
isSparkSessionManaged, isSparkSessionManaged,
spark -> { spark -> {
prepareResults(spark, inputPath, outputPath); prepareResults(spark, inputPath, outputPath);
}); });
} }
private static <I extends Result> void prepareResults(SparkSession spark, String inputPath, String outputPath) { private static <I extends Result> void prepareResults(SparkSession spark, String inputPath, String outputPath) {
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); JavaRDD<BipDeserialize> bipDeserializeJavaRDD = sc
.textFile(inputPath)
.map(item -> OBJECT_MAPPER.readValue(item, BipDeserialize.class));
JavaRDD<BipDeserialize> bipDeserializeJavaRDD = sc spark
.textFile(inputPath) .createDataset(bipDeserializeJavaRDD.flatMap(entry -> entry.keySet().stream().map(key -> {
.map(item -> OBJECT_MAPPER.readValue(item, BipDeserialize.class)); BipScore bs = new BipScore();
bs.setId(getIdentifier(key));
spark bs.setScoreList(entry.get(key));
.createDataset(bipDeserializeJavaRDD.flatMap(entry -> entry.keySet().stream().map(key -> { return bs;
BipScore bs = new BipScore(); }).collect(Collectors.toList()).iterator()).rdd(), Encoders.bean(BipScore.class))
bs.setId(getIdentifier(key)); .write()
bs.setScoreList(entry.get(key)); .mode(SaveMode.Overwrite)
return bs; .option("compression", "gzip")
}).collect(Collectors.toList()).iterator()).rdd(), Encoders.bean(BipScore.class)) .json(outputPath);
.write() }
.mode(SaveMode.Overwrite)
.option("compression","gzip")
.json(outputPath);
}
} }

84
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bypassactionset/bipfinder/SparkUpdateBip.java
View File

@ -1,45 +1,44 @@
package eu.dnetlib.dhp.bypassactionset.bipfinder; package eu.dnetlib.dhp.bypassactionset.bipfinder;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import static eu.dnetlib.dhp.PropagationConstant.*;
import eu.dnetlib.dhp.bypassactionset.model.BipScore; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.*;
import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2;
import java.io.Serializable; import java.io.Serializable;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static eu.dnetlib.dhp.PropagationConstant.*; import org.apache.commons.io.IOUtils;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.bypassactionset.model.BipScore;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.*;
import scala.Tuple2;
/** /**
* created the Atomic Action for each tipe of results * created the Atomic Action for each tipe of results
*/ */
public class SparkUpdateBip implements Serializable { public class SparkUpdateBip implements Serializable {
private static final Logger log = LoggerFactory.getLogger(SparkUpdateBip.class); private static final Logger log = LoggerFactory.getLogger(SparkUpdateBip.class);
public static <I extends Result> void main(String[] args) throws Exception { public static <I extends Result> void main(String[] args) throws Exception {
String jsonConfiguration = IOUtils String jsonConfiguration = IOUtils
.toString( .toString(
SparkUpdateBip.class SparkUpdateBip.class
.getResourceAsStream( .getResourceAsStream(
"/eu/dnetlib/dhp/actionmanager/bipfinder/input_parameters.json")); "/eu/dnetlib/dhp/bypassactionset/bip_update_parameters.json"));
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
@ -71,10 +70,9 @@ public class SparkUpdateBip implements Serializable {
runWithSparkSession( runWithSparkSession(
conf, conf,
isSparkSessionManaged, isSparkSessionManaged,
spark -> spark -> updateBipFinder(spark, inputPath, outputPath, bipScorePath, inputClazz)
updateBipFinder(spark, inputPath, outputPath, bipScorePath, inputClazz)
); );
} }
private static <I extends Result> void updateBipFinder(SparkSession spark, String inputPath, String outputPath, private static <I extends Result> void updateBipFinder(SparkSession spark, String inputPath, String outputPath,
@ -83,18 +81,19 @@ public class SparkUpdateBip implements Serializable {
Dataset<I> results = readPath(spark, inputPath, inputClazz); Dataset<I> results = readPath(spark, inputPath, inputClazz);
Dataset<BipScore> bipScores = readPath(spark, bipScorePath, BipScore.class); Dataset<BipScore> bipScores = readPath(spark, bipScorePath, BipScore.class);
results.joinWith(bipScores, results.col("id").equalTo(bipScores.col("id")), "left") results
.map((MapFunction<Tuple2<I,BipScore>, I>) value -> { .joinWith(bipScores, results.col("id").equalTo(bipScores.col("id")), "left")
if (!Optional.ofNullable(value._2()).isPresent()){ .map((MapFunction<Tuple2<I, BipScore>, I>) value -> {
return value._1(); if (!Optional.ofNullable(value._2()).isPresent()) {
}
value._1().setMeasures(getMeasure(value._2()));
return value._1(); return value._1();
}, Encoders.bean(inputClazz)) }
.write() value._1().setMeasures(getMeasure(value._2()));
.mode(SaveMode.Overwrite) return value._1();
.option("compression","gzip") }, Encoders.bean(inputClazz))
.json(outputPath + "/bip"); .write()
.mode(SaveMode.Overwrite)
.option("compression", "gzip")
.json(outputPath + "/bip");
} }
@ -114,10 +113,13 @@ public class SparkUpdateBip implements Serializable {
KeyValue kv = new KeyValue(); KeyValue kv = new KeyValue();
kv.setValue(unit.getValue()); kv.setValue(unit.getValue());
kv.setKey(unit.getKey()); kv.setKey(unit.getKey());
kv.setDataInfo(getDataInfo(UPDATE_DATA_INFO_TYPE, kv
UPDATE_MEASURE_BIP_CLASS_ID, .setDataInfo(
UPDATE_MEASURE_BIP_CLASS_NAME, getDataInfo(
ModelConstants.DNET_PROVENANCE_ACTIONS, "")); UPDATE_DATA_INFO_TYPE,
UPDATE_MEASURE_BIP_CLASS_ID,
UPDATE_CLASS_NAME,
ModelConstants.DNET_PROVENANCE_ACTIONS, ""));
return kv; return kv;
}) })
.collect(Collectors.toList())); .collect(Collectors.toList()));
@ -126,10 +128,4 @@ public class SparkUpdateBip implements Serializable {
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
} }

94
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bypassactionset/fos/GetFOSData.java
View File

@ -1,8 +1,10 @@
package eu.dnetlib.dhp.bypassactionset.fos; package eu.dnetlib.dhp.bypassactionset.fos;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import java.io.*;
import java.util.Objects;
import java.util.Optional;
import eu.dnetlib.dhp.common.collection.GetCSV;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
@ -10,70 +12,64 @@ import org.apache.hadoop.fs.Path;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.*; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.collection.GetCSV;
import java.util.Objects;
import java.util.Optional;
public class GetFOSData implements Serializable { public class GetFOSData implements Serializable {
private static final Logger log = LoggerFactory.getLogger(GetFOSData.class); private static final Logger log = LoggerFactory.getLogger(GetFOSData.class);
public static final char DEFAULT_DELIMITER = '\t'; public static final char DEFAULT_DELIMITER = '\t';
public static void main(final String[] args) throws Exception { public static void main(final String[] args) throws Exception {
final ArgumentApplicationParser parser = new ArgumentApplicationParser( final ArgumentApplicationParser parser = new ArgumentApplicationParser(
IOUtils IOUtils
.toString( .toString(
Objects Objects
.requireNonNull( .requireNonNull(
GetFOSData.class GetFOSData.class
.getResourceAsStream( .getResourceAsStream(
"/eu/dnetlib/dhp/bypassactionset/get_fos_parameters.json")))); "/eu/dnetlib/dhp/bypassactionset/get_fos_parameters.json"))));
parser.parseArgument(args); parser.parseArgument(args);
//the path where the original fos csv file is stored // the path where the original fos csv file is stored
final String inputPath = parser.get("inputPath"); final String inputPath = parser.get("inputPath");
log.info("inputPath {}", inputPath); log.info("inputPath {}", inputPath);
//the path where to put the file as json // the path where to put the file as json
final String outputFile = parser.get("outputFile"); final String outputFile = parser.get("outputFile");
log.info("outputFile {}", outputFile); log.info("outputFile {}", outputFile);
final String hdfsNameNode = parser.get("hdfsNameNode"); final String hdfsNameNode = parser.get("hdfsNameNode");
log.info("hdfsNameNode {}", hdfsNameNode); log.info("hdfsNameNode {}", hdfsNameNode);
final String classForName = parser.get("classForName"); final String classForName = parser.get("classForName");
log.info("classForName {}", classForName); log.info("classForName {}", classForName);
final char delimiter = Optional final char delimiter = Optional
.ofNullable(parser.get("delimiter")) .ofNullable(parser.get("delimiter"))
.map(s -> s.charAt(0)) .map(s -> s.charAt(0))
.orElse(DEFAULT_DELIMITER); .orElse(DEFAULT_DELIMITER);
log.info("delimiter {}", delimiter); log.info("delimiter {}", delimiter);
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfsNameNode); conf.set("fs.defaultFS", hdfsNameNode);
FileSystem fileSystem = FileSystem.get(conf); FileSystem fileSystem = FileSystem.get(conf);
new GetFOSData().doRewrite(inputPath, outputFile, classForName, delimiter, fileSystem); new GetFOSData().doRewrite(inputPath, outputFile, classForName, delimiter, fileSystem);
} }
public void doRewrite(String inputPath, String outputFile, String classForName, char delimiter, FileSystem fs) public void doRewrite(String inputPath, String outputFile, String classForName, char delimiter, FileSystem fs)
throws IOException, ClassNotFoundException{ throws IOException, ClassNotFoundException {
//reads the csv and writes it as its json equivalent
try (InputStreamReader reader = new InputStreamReader(fs.open(new Path (inputPath)))) {
GetCSV.getCsv(fs, reader, outputFile, classForName, delimiter);
}
}
// reads the csv and writes it as its json equivalent
try (InputStreamReader reader = new InputStreamReader(fs.open(new Path(inputPath)))) {
GetCSV.getCsv(fs, reader, outputFile, classForName, delimiter);
}
}
} }

122
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bypassactionset/fos/PrepareFOSSparkJob.java
View File

@ -1,9 +1,15 @@
package eu.dnetlib.dhp.bypassactionset.fos; package eu.dnetlib.dhp.bypassactionset.fos;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import static eu.dnetlib.dhp.PropagationConstant.*;
import eu.dnetlib.dhp.bypassactionset.model.FOSDataModel; import static eu.dnetlib.dhp.bypassactionset.Utils.getIdentifier;
import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.FlatMapFunction;
@ -15,78 +21,66 @@ import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.Serializable; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import java.util.ArrayList; import eu.dnetlib.dhp.bypassactionset.model.FOSDataModel;
import java.util.Arrays; import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
import java.util.List; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
import static eu.dnetlib.dhp.PropagationConstant.*;
import static eu.dnetlib.dhp.bypassactionset.Utils.getIdentifier;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
public class PrepareFOSSparkJob implements Serializable { public class PrepareFOSSparkJob implements Serializable {
private static final Logger log = LoggerFactory.getLogger(PrepareFOSSparkJob.class); private static final Logger log = LoggerFactory.getLogger(PrepareFOSSparkJob.class);
public static void main(String[] args) throws Exception {
public static void main(String[] args) throws Exception { String jsonConfiguration = IOUtils
.toString(
PrepareFOSSparkJob.class
.getResourceAsStream(
"/eu/dnetlib/dhp/bypassactionset/distribute_fos_parameters.json"));
String jsonConfiguration = IOUtils final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
.toString(
PrepareFOSSparkJob.class
.getResourceAsStream(
"/eu/dnetlib/dhp/bypassactionset/distribute_fos_parameters.json"));
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); parser.parseArgument(args);
parser.parseArgument(args); Boolean isSparkSessionManaged = isSparkSessionManaged(parser);
log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
Boolean isSparkSessionManaged = isSparkSessionManaged(parser); String sourcePath = parser.get("sourcePath");
log.info("isSparkSessionManaged: {}", isSparkSessionManaged); log.info("sourcePath: {}", sourcePath);
String sourcePath = parser.get("sourcePath"); final String outputPath = parser.get("outputPath");
log.info("sourcePath: {}", sourcePath); log.info("outputPath: {}", outputPath);
SparkConf conf = new SparkConf();
runWithSparkSession(
conf,
isSparkSessionManaged,
spark -> {
removeOutputDir(spark, outputPath);
distributeFOSdois(
spark,
sourcePath,
outputPath);
});
}
final String outputPath = parser.get("outputPath"); private static void distributeFOSdois(SparkSession spark, String sourcePath, String outputPath) {
log.info("outputPath: {}", outputPath); Dataset<FOSDataModel> fosDataset = readPath(spark, sourcePath, FOSDataModel.class);
SparkConf conf = new SparkConf();
runWithSparkSession(
conf,
isSparkSessionManaged,
spark -> {
removeOutputDir(spark, outputPath);
distributeFOSdois(
spark,
sourcePath,
outputPath
);
});
}
private static void distributeFOSdois(SparkSession spark, String sourcePath, String outputPath) {
Dataset<FOSDataModel> fosDataset = readPath(spark, sourcePath, FOSDataModel.class);
fosDataset.flatMap((FlatMapFunction<FOSDataModel, FOSDataModel>) v -> {
List<FOSDataModel> fosList = new ArrayList<>();
final String level1 = v.getLevel1();
final String level2 = v.getLevel2();
final String level3 = v.getLevel3();
Arrays.stream(v.getDoi().split("\u0002")).forEach(d ->
fosList.add(FOSDataModel.newInstance(getIdentifier(d), level1, level2, level3)));
return fosList.iterator();
}, Encoders.bean(FOSDataModel.class))
.write()
.mode(SaveMode.Overwrite)
.option("compression","gzip")
.json(outputPath);
}
fosDataset.flatMap((FlatMapFunction<FOSDataModel, FOSDataModel>) v -> {
List<FOSDataModel> fosList = new ArrayList<>();
final String level1 = v.getLevel1();
final String level2 = v.getLevel2();
final String level3 = v.getLevel3();
Arrays
.stream(v.getDoi().split("\u0002"))
.forEach(d -> fosList.add(FOSDataModel.newInstance(getIdentifier(d), level1, level2, level3)));
return fosList.iterator();
}, Encoders.bean(FOSDataModel.class))
.write()
.mode(SaveMode.Overwrite)
.option("compression", "gzip")
.json(outputPath);
}
} }

163
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bypassactionset/fos/SparkUpdateFOS.java
View File

@ -1,11 +1,13 @@
package eu.dnetlib.dhp.bypassactionset.fos; package eu.dnetlib.dhp.bypassactionset.fos;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import static eu.dnetlib.dhp.PropagationConstant.*;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import java.io.Serializable;
import java.util.*;
import java.util.stream.Collectors;
import eu.dnetlib.dhp.bypassactionset.model.FOSDataModel;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.Result;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.MapFunction;
@ -15,104 +17,105 @@ import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.bypassactionset.model.FOSDataModel;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.Result;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
import scala.Tuple2; import scala.Tuple2;
import java.io.Serializable;
import java.util.*;
import java.util.stream.Collectors;
import static eu.dnetlib.dhp.PropagationConstant.*;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
public class SparkUpdateFOS implements Serializable { public class SparkUpdateFOS implements Serializable {
private static final Logger log = LoggerFactory.getLogger(SparkUpdateFOS.class); private static final Logger log = LoggerFactory.getLogger(SparkUpdateFOS.class);
public static <I extends Result> void main(String[] args) throws Exception {
public static <I extends Result> void main(String[] args) throws Exception { String jsonConfiguration = IOUtils
.toString(
SparkUpdateFOS.class
.getResourceAsStream(
"/eu/dnetlib/dhp/bypassactionset/fos_update_parameters.json"));
String jsonConfiguration = IOUtils final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
.toString(
SparkUpdateFOS.class
.getResourceAsStream(
"/eu/dnetlib/dhp/actionmanager/bipfinder/input_parameters.json"));
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); parser.parseArgument(args);
parser.parseArgument(args); Boolean isSparkSessionManaged = Optional
.ofNullable(parser.get("isSparkSessionManaged"))
.map(Boolean::valueOf)
.orElse(Boolean.TRUE);
Boolean isSparkSessionManaged = Optional log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
.ofNullable(parser.get("isSparkSessionManaged"))
.map(Boolean::valueOf)
.orElse(Boolean.TRUE);
log.info("isSparkSessionManaged: {}", isSparkSessionManaged); final String inputPath = parser.get("inputPath");
log.info("inputPath {}: ", inputPath);
final String inputPath = parser.get("inputPath"); final String outputPath = parser.get("outputPath");
log.info("inputPath {}: ", inputPath); log.info("outputPath {}: ", outputPath);
final String outputPath = parser.get("outputPath"); final String fosPath = parser.get("fosPath");
log.info("outputPath {}: ", outputPath); log.info("fosPath: {}", fosPath);
final String fosPath = parser.get("fosPath"); final String resultClassName = parser.get("resultTableName");
log.info("fosPath: {}", fosPath); log.info("resultTableName: {}", resultClassName);
final String resultClassName = parser.get("resultTableName"); Class<I> inputClazz = (Class<I>) Class.forName(resultClassName);
log.info("resultTableName: {}", resultClassName);
Class<I> inputClazz = (Class<I>) Class.forName(resultClassName); SparkConf conf = new SparkConf();
SparkConf conf = new SparkConf(); runWithSparkSession(
conf,
isSparkSessionManaged,
spark -> updateFos(spark, inputPath, outputPath, fosPath, inputClazz)
runWithSparkSession( );
conf, }
isSparkSessionManaged,
spark ->
updateFos(spark, inputPath, outputPath, fosPath, inputClazz)
); private static <I extends Result> void updateFos(SparkSession spark, String inputPath, String outputPath,
} String fosPath, Class<I> inputClazz) {
private static <I extends Result> void updateFos(SparkSession spark, String inputPath, String outputPath, Dataset<I> results = readPath(spark, inputPath, inputClazz);
String bipScorePath, Class<I> inputClazz) { Dataset<FOSDataModel> fosDataModelDataset = readPath(spark, fosPath, FOSDataModel.class);
Dataset<I> results = readPath(spark, inputPath, inputClazz); results
Dataset<FOSDataModel> bipScores = readPath(spark, bipScorePath, FOSDataModel.class); .joinWith(fosDataModelDataset, results.col("id").equalTo(fosDataModelDataset.col("doi")), "left")
.map((MapFunction<Tuple2<I, FOSDataModel>, I>) value -> {
if (!Optional.ofNullable(value._2()).isPresent()) {
return value._1();
}
value._1().getSubject().addAll(getSubjects(value._2()));
return value._1();
}, Encoders.bean(inputClazz))
.write()
.mode(SaveMode.Overwrite)
.option("compression", "gzip")
.json(outputPath);
results.joinWith(bipScores, results.col("id").equalTo(bipScores.col("id")), "left") }
.map((MapFunction<Tuple2<I,FOSDataModel>, I>) value -> {
if (!Optional.ofNullable(value._2()).isPresent()){
return value._1();
}
value._1().getSubject().addAll(getSubjects(value._2()));
return value._1();
}, Encoders.bean(inputClazz))
.write()
.mode(SaveMode.Overwrite)
.option("compression","gzip")
.json(outputPath);
} private static List<StructuredProperty> getSubjects(FOSDataModel fos) {
return Arrays
.asList(getSubject(fos.getLevel1()), getSubject(fos.getLevel2()), getSubject(fos.getLevel3()))
.stream()
.filter(Objects::nonNull)
.collect(Collectors.toList());
}
private static List<StructuredProperty> getSubjects(FOSDataModel fos) { private static StructuredProperty getSubject(String sbj) {
return Arrays.asList(getSubject(fos.getLevel1()), getSubject(fos.getLevel2()), getSubject(fos.getLevel3())) if (sbj.equals(NULL))
.stream() return null;
.filter(Objects::nonNull) StructuredProperty sp = new StructuredProperty();
.collect(Collectors.toList()); sp.setValue(sbj);
} sp.setQualifier(getQualifier(FOS_CLASS_ID, FOS_CLASS_NAME, ModelConstants.DNET_SUBJECT_TYPOLOGIES));
sp
private static StructuredProperty getSubject(String sbj) { .setDataInfo(
if (sbj.equals(NULL)) getDataInfo(
return null; UPDATE_DATA_INFO_TYPE,
StructuredProperty sp = new StructuredProperty(); UPDATE_SUBJECT_FOS_CLASS_ID,
sp.setValue(sbj); UPDATE_CLASS_NAME,
sp.setQualifier(getQualifier(FOS_CLASS_ID, FOS_CLASS_NAME, ModelConstants.DNET_SUBJECT_TYPOLOGIES)); ModelConstants.DNET_PROVENANCE_ACTIONS, ""));
sp.setDataInfo(getDataInfo(UPDATE_DATA_INFO_TYPE, return sp;
UPDATE_SUBJECT_FOS_CLASS_ID,
UPDATE_SUBJECT_FOS_CLASS_NAME,
ModelConstants.DNET_PROVENANCE_ACTIONS, ""));
return sp;
}
}
} }

92
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bypassactionset/model/FOSDataModel.java
View File

@ -1,71 +1,71 @@
package eu.dnetlib.dhp.bypassactionset.model;
import com.opencsv.bean.CsvBindByPosition; package eu.dnetlib.dhp.bypassactionset.model;
import java.io.Serializable; import java.io.Serializable;
import com.opencsv.bean.CsvBindByPosition;
public class FOSDataModel implements Serializable { public class FOSDataModel implements Serializable {
@CsvBindByPosition(position = 1) @CsvBindByPosition(position = 1)
// @CsvBindByName(column = "doi") // @CsvBindByName(column = "doi")
private String doi; private String doi;
@CsvBindByPosition(position = 2) @CsvBindByPosition(position = 2)
// @CsvBindByName(column = "level1") // @CsvBindByName(column = "level1")
private String level1; private String level1;
@CsvBindByPosition(position = 3) @CsvBindByPosition(position = 3)
// @CsvBindByName(column = "level2") // @CsvBindByName(column = "level2")
private String level2; private String level2;
@CsvBindByPosition(position = 4) @CsvBindByPosition(position = 4)
// @CsvBindByName(column = "level3") // @CsvBindByName(column = "level3")
private String level3; private String level3;
public FOSDataModel() {
public FOSDataModel() { }
} public FOSDataModel(String doi, String level1, String level2, String level3) {
this.doi = doi;
this.level1 = level1;
this.level2 = level2;
this.level3 = level3;
}
public FOSDataModel(String doi, String level1, String level2, String level3) { public static FOSDataModel newInstance(String d, String level1, String level2, String level3) {
this.doi = doi; return new FOSDataModel(d, level1, level2, level3);
this.level1 = level1; }
this.level2 = level2;
this.level3 = level3;
}
public static FOSDataModel newInstance(String d, String level1, String level2, String level3) { public String getDoi() {
return new FOSDataModel(d, level1, level2, level3); return doi;
} }
public String getDoi() { public void setDoi(String doi) {
return doi; this.doi = doi;
} }
public void setDoi(String doi) { public String getLevel1() {
this.doi = doi; return level1;
} }
public String getLevel1() { public void setLevel1(String level1) {
return level1; this.level1 = level1;
} }
public void setLevel1(String level1) { public String getLevel2() {
this.level1 = level1; return level2;
} }
public String getLevel2() { public void setLevel2(String level2) {
return level2; this.level2 = level2;
} }
public void setLevel2(String level2) { public String getLevel3() {
this.level2 = level2; return level3;
} }
public String getLevel3() { public void setLevel3(String level3) {
return level3; this.level3 = level3;
} }
public void setLevel3(String level3) {
this.level3 = level3;
}
} }

95
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bypassactionset/opencitations/GetOpenCitationsRefs.java
View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.bypassactionset.opencitations; package eu.dnetlib.dhp.bypassactionset.opencitations;
import java.io.*; import java.io.*;
@ -20,73 +21,73 @@ import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
public class GetOpenCitationsRefs implements Serializable { public class GetOpenCitationsRefs implements Serializable {
private static final Logger log = LoggerFactory.getLogger(GetOpenCitationsRefs.class); private static final Logger log = LoggerFactory.getLogger(GetOpenCitationsRefs.class);
public static void main(final String[] args) throws IOException, ParseException { public static void main(final String[] args) throws IOException, ParseException {
final ArgumentApplicationParser parser = new ArgumentApplicationParser( final ArgumentApplicationParser parser = new ArgumentApplicationParser(
IOUtils IOUtils
.toString( .toString(
Objects Objects
.requireNonNull( .requireNonNull(
GetOpenCitationsRefs.class GetOpenCitationsRefs.class
.getResourceAsStream( .getResourceAsStream(
"/eu/dnetlib/dhp/bypassactionset/opencitations/input_parameters.json")))); "/eu/dnetlib/dhp/bypassactionset/opencitations/input_parameters.json"))));
parser.parseArgument(args); parser.parseArgument(args);
final String[] inputFile = parser.get("inputFile").split(";"); final String[] inputFile = parser.get("inputFile").split(";");
log.info("inputFile {}", inputFile.toString()); log.info("inputFile {}", inputFile.toString());
final String workingPath = parser.get("workingPath"); final String workingPath = parser.get("workingPath");
log.info("workingPath {}", workingPath); log.info("workingPath {}", workingPath);
final String hdfsNameNode = parser.get("hdfsNameNode"); final String hdfsNameNode = parser.get("hdfsNameNode");
log.info("hdfsNameNode {}", hdfsNameNode); log.info("hdfsNameNode {}", hdfsNameNode);
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfsNameNode); conf.set("fs.defaultFS", hdfsNameNode);
FileSystem fileSystem = FileSystem.get(conf); FileSystem fileSystem = FileSystem.get(conf);
GetOpenCitationsRefs ocr = new GetOpenCitationsRefs(); GetOpenCitationsRefs ocr = new GetOpenCitationsRefs();
for (String file : inputFile) { for (String file : inputFile) {
ocr.doExtract(workingPath + "/Original/" + file, workingPath, fileSystem); ocr.doExtract(workingPath + "/Original/" + file, workingPath, fileSystem);
} }
} }
private void doExtract(String inputFile, String workingPath, FileSystem fileSystem) private void doExtract(String inputFile, String workingPath, FileSystem fileSystem)
throws IOException { throws IOException {
final Path path = new Path(inputFile); final Path path = new Path(inputFile);
FSDataInputStream oc_zip = fileSystem.open(path); FSDataInputStream oc_zip = fileSystem.open(path);
int count = 1; int count = 1;
try (ZipInputStream zis = new ZipInputStream(oc_zip)) { try (ZipInputStream zis = new ZipInputStream(oc_zip)) {
ZipEntry entry = null; ZipEntry entry = null;
while ((entry = zis.getNextEntry()) != null) { while ((entry = zis.getNextEntry()) != null) {
if (!entry.isDirectory()) { if (!entry.isDirectory()) {
String fileName = entry.getName(); String fileName = entry.getName();
fileName = fileName.substring(0, fileName.indexOf("T")) + "_" + count; fileName = fileName.substring(0, fileName.indexOf("T")) + "_" + count;
count++; count++;
try ( try (
FSDataOutputStream out = fileSystem FSDataOutputStream out = fileSystem
.create(new Path(workingPath + "/COCI/" + fileName + ".gz")); .create(new Path(workingPath + "/COCI/" + fileName + ".gz"));
GZIPOutputStream gzipOs = new GZIPOutputStream(new BufferedOutputStream(out))) { GZIPOutputStream gzipOs = new GZIPOutputStream(new BufferedOutputStream(out))) {
IOUtils.copy(zis, gzipOs); IOUtils.copy(zis, gzipOs);
} }
} }
} }
} }
} }
} }

200
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bypassactionset/opencitations/SparkUpdateOCRels.java
View File

@ -1,5 +1,5 @@
package eu.dnetlib.dhp.bypassactionset.opencitations;
package eu.dnetlib.dhp.bypassactionset.opencitations;
import static eu.dnetlib.dhp.PropagationConstant.*; import static eu.dnetlib.dhp.PropagationConstant.*;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
@ -10,7 +10,6 @@ import java.util.*;
import org.apache.commons.cli.ParseException; import org.apache.commons.cli.ParseException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.FilterFunction;
import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.FlatMapFunction;
@ -23,136 +22,129 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.*;
import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
public class SparkUpdateOCRels implements Serializable { public class SparkUpdateOCRels implements Serializable {
private static final Logger log = LoggerFactory.getLogger(SparkUpdateOCRels.class); private static final Logger log = LoggerFactory.getLogger(SparkUpdateOCRels.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
public static void main(final String[] args) throws IOException, ParseException { public static void main(final String[] args) throws IOException, ParseException {
final ArgumentApplicationParser parser = new ArgumentApplicationParser( final ArgumentApplicationParser parser = new ArgumentApplicationParser(
IOUtils IOUtils
.toString( .toString(
Objects Objects
.requireNonNull( .requireNonNull(
SparkUpdateOCRels.class SparkUpdateOCRels.class
.getResourceAsStream( .getResourceAsStream(
"/eu/dnetlib/dhp/actionmanager/opencitations/as_parameters.json")))); "/eu/dnetlib/dhp/actionmanager/opencitations/as_parameters.json"))));
parser.parseArgument(args); parser.parseArgument(args);
Boolean isSparkSessionManaged = Optional Boolean isSparkSessionManaged = Optional
.ofNullable(parser.get("isSparkSessionManaged")) .ofNullable(parser.get("isSparkSessionManaged"))
.map(Boolean::valueOf) .map(Boolean::valueOf)
.orElse(Boolean.TRUE); .orElse(Boolean.TRUE);
log.info("isSparkSessionManaged: {}", isSparkSessionManaged); log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
final String inputPath = parser.get("inputPath"); final String inputPath = parser.get("inputPath");
log.info("inputPath {}", inputPath.toString()); log.info("inputPath {}", inputPath.toString());
final String outputPath = parser.get("outputPath"); final String outputPath = parser.get("outputPath");
log.info("outputPath {}", outputPath); log.info("outputPath {}", outputPath);
final boolean shouldDuplicateRels = final boolean shouldDuplicateRels = Optional
Optional.ofNullable(parser.get("shouldDuplicateRels")) .ofNullable(parser.get("shouldDuplicateRels"))
.map(Boolean::valueOf) .map(Boolean::valueOf)
.orElse(Boolean.FALSE); .orElse(Boolean.FALSE);
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();
runWithSparkSession( runWithSparkSession(
conf, conf,
isSparkSessionManaged, isSparkSessionManaged,
spark -> spark -> addOCRelations(spark, inputPath, outputPath, shouldDuplicateRels));
addOCRelations(spark, inputPath, outputPath, shouldDuplicateRels)
);
} }
private static void addOCRelations(SparkSession spark, String inputPath, String outputPath, private static void addOCRelations(SparkSession spark, String inputPath, String outputPath,
boolean shouldDuplicateRels) { boolean shouldDuplicateRels) {
spark spark
.sqlContext() .sqlContext()
.createDataset(spark.sparkContext().textFile(inputPath + "/*", 6000), Encoders.STRING()) .createDataset(spark.sparkContext().textFile(inputPath + "/*", 6000), Encoders.STRING())
.flatMap( .flatMap(
(FlatMapFunction<String, Relation>) value -> createRelation(value, shouldDuplicateRels).iterator(), (FlatMapFunction<String, Relation>) value -> createRelation(value, shouldDuplicateRels).iterator(),
Encoders.bean(Relation.class)) Encoders.bean(Relation.class))
.filter((FilterFunction<Relation>) value -> value != null) .filter((FilterFunction<Relation>) value -> value != null)
.write() .write()
.mode(SaveMode.Append) .mode(SaveMode.Append)
.option("compression", "gzip") .option("compression", "gzip")
.json(outputPath); .json(outputPath);
} }
private static List<Relation> createRelation(String value, boolean duplicate) { private static List<Relation> createRelation(String value, boolean duplicate) {
String[] line = value.split(","); String[] line = value.split(",");
if (!line[1].startsWith("10.")) { if (!line[1].startsWith("10.")) {
return new ArrayList<>(); return new ArrayList<>();
} }
List<Relation> relationList = new ArrayList<>(); List<Relation> relationList = new ArrayList<>();
String citing = ID_PREFIX + IdentifierFactory.md5(CleaningFunctions.normalizePidValue(DOI, line[1])); String citing = ID_PREFIX + IdentifierFactory.md5(CleaningFunctions.normalizePidValue(DOI, line[1]));
final String cited = ID_PREFIX + IdentifierFactory.md5(CleaningFunctions.normalizePidValue(DOI, line[2])); final String cited = ID_PREFIX + IdentifierFactory.md5(CleaningFunctions.normalizePidValue(DOI, line[2]));
relationList relationList
.addAll( .addAll(
getRelations( getRelations(
citing, citing,
cited)); cited));
if (duplicate && line[1].endsWith(REF_DOI)) { if (duplicate && line[1].endsWith(REF_DOI)) {
citing = ID_PREFIX + IdentifierFactory citing = ID_PREFIX + IdentifierFactory
.md5(CleaningFunctions.normalizePidValue(DOI, line[1].substring(0, line[1].indexOf(REF_DOI)))); .md5(CleaningFunctions.normalizePidValue(DOI, line[1].substring(0, line[1].indexOf(REF_DOI))));
relationList.addAll(getRelations(citing, cited)); relationList.addAll(getRelations(citing, cited));
} }
return relationList; return relationList;
} }
private static Collection<Relation> getRelations(String citing, String cited) { private static Collection<Relation> getRelations(String citing, String cited) {
return Arrays return Arrays
.asList( .asList(
getRelation(citing, cited, ModelConstants.CITES), getRelation(citing, cited, ModelConstants.CITES),
getRelation(cited, citing, ModelConstants.IS_CITED_BY)); getRelation(cited, citing, ModelConstants.IS_CITED_BY));
} }
public static Relation getRelation( public static Relation getRelation(
String source, String source,
String target, String target,
String relclass) { String relclass) {
Relation r = new Relation(); Relation r = new Relation();
r.setCollectedfrom(getCollectedFrom()); r.setCollectedfrom(getCollectedFrom());
r.setSource(source); r.setSource(source);
r.setTarget(target); r.setTarget(target);
r.setRelClass(relclass); r.setRelClass(relclass);
r.setRelType(ModelConstants.RESULT_RESULT); r.setRelType(ModelConstants.RESULT_RESULT);
r.setSubRelType(ModelConstants.CITATION); r.setSubRelType(ModelConstants.CITATION);
r r
.setDataInfo( .setDataInfo(
getDataInfo(UPDATE_DATA_INFO_TYPE, OPENCITATIONS_CLASSID, OPENCITATIONS_CLASSNAME, getDataInfo(
ModelConstants.DNET_PROVENANCE_ACTIONS, OC_TRUST, false)); UPDATE_DATA_INFO_TYPE, OPENCITATIONS_CLASSID, OPENCITATIONS_CLASSNAME,
return r; ModelConstants.DNET_PROVENANCE_ACTIONS, OC_TRUST, false));
} return r;
}
public static List<KeyValue> getCollectedFrom() {
KeyValue kv = new KeyValue();
kv.setKey(ModelConstants.OPENOCITATIONS_ID);
kv.setValue(ModelConstants.OPENOCITATIONS_NAME);
return Arrays.asList(kv);
}
public static List<KeyValue> getCollectedFrom() {
KeyValue kv = new KeyValue();
kv.setKey(ModelConstants.OPENOCITATIONS_ID);
kv.setValue(ModelConstants.OPENOCITATIONS_NAME);
return Arrays.asList(kv);
}
} }

20
dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bypassactionset/bip_prepare_parameters.json Normal file
View File

@ -0,0 +1,20 @@
[
{
"paramName": "issm",
"paramLongName": "isSparkSessionManaged",
"paramDescription": "when true will stop SparkSession after job execution",
"paramRequired": false
},
{
"paramName": "ip",
"paramLongName": "inputPath",
"paramDescription": "the URL from where to get the programme file",
"paramRequired": true
},
{
"paramName": "o",
"paramLongName": "outputPath",
"paramDescription": "the path of the new ActionSet",
"paramRequired": true
}
]

32
dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bypassactionset/bip_update_parameters.json Normal file
View File

@ -0,0 +1,32 @@
[
{
"paramName": "issm",
"paramLongName": "isSparkSessionManaged",
"paramDescription": "when true will stop SparkSession after job execution",
"paramRequired": false
},
{
"paramName": "ip",
"paramLongName": "inputPath",
"paramDescription": "the URL from where to get the programme file",
"paramRequired": true
},
{
"paramName": "o",
"paramLongName": "outputPath",
"paramDescription": "the path of the new ActionSet",
"paramRequired": true
},
{
"paramName": "rtn",
"paramLongName": "resultTableName",
"paramDescription": "the path of the new ActionSet",
"paramRequired": true
},
{
"paramName": "bsp",
"paramLongName": "bipScorePath",
"paramDescription": "the path of the new ActionSet",
"paramRequired": true
}
]

32
dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bypassactionset/fos_update_parameters.json Normal file
View File

@ -0,0 +1,32 @@
[
{
"paramName": "issm",
"paramLongName": "isSparkSessionManaged",
"paramDescription": "when true will stop SparkSession after job execution",
"paramRequired": false
},
{
"paramName": "ip",
"paramLongName": "inputPath",
"paramDescription": "the URL from where to get the programme file",
"paramRequired": true
},
{
"paramName": "o",
"paramLongName": "outputPath",
"paramDescription": "the path of the new ActionSet",
"paramRequired": true
},
{
"paramName": "rtn",
"paramLongName": "resultTableName",
"paramDescription": "the path of the new ActionSet",
"paramRequired": true
},
{
"paramName": "fp",
"paramLongName": "fosPath",
"paramDescription": "the path of the new ActionSet",
"paramRequired": true
}
]

250
dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bypassactionset/BipTest.java Normal file
View File

@ -0,0 +1,250 @@
package eu.dnetlib.dhp.bypassactionset;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
import eu.dnetlib.dhp.schema.oaf.Author;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.neethi.Assertion;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.bypassactionset.bipfinder.PrepareBipFinder;
import eu.dnetlib.dhp.bypassactionset.bipfinder.SparkUpdateBip;
import eu.dnetlib.dhp.bypassactionset.model.BipScore;
import eu.dnetlib.dhp.countrypropagation.CountryPropagationJobTest;
import eu.dnetlib.dhp.schema.oaf.Measure;
import eu.dnetlib.dhp.schema.oaf.Publication;
import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
public class BipTest {
private static final Logger log = LoggerFactory.getLogger(FOSTest.class);
private static Path workingDir;
private static SparkSession spark;
private static LocalFileSystem fs;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String ID_PREFIX = "50|doi_________";
@BeforeAll
public static void beforeAll() throws IOException {
workingDir = Files.createTempDirectory(BipTest.class.getSimpleName());
fs = FileSystem.getLocal(new Configuration());
log.info("using work dir {}", workingDir);
SparkConf conf = new SparkConf();
conf.setAppName(FOSTest.class.getSimpleName());
conf.setMaster("local[*]");
conf.set("spark.driver.host", "localhost");
conf.set("hive.metastore.local", "true");
conf.set("spark.ui.enabled", "false");
conf.set("spark.sql.warehouse.dir", workingDir.toString());
conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
spark = SparkSession
.builder()
.appName(CountryPropagationJobTest.class.getSimpleName())
.config(conf)
.getOrCreate();
}
@AfterAll
public static void afterAll() throws IOException {
FileUtils.deleteDirectory(workingDir.toFile());
spark.stop();
}
@Test
void prepareBipTest() throws Exception {
final String sourcePath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/bip/bip.json")
.getPath();
PrepareBipFinder
.main(
new String[] {
"--isSparkSessionManaged", Boolean.FALSE.toString(),
"--inputPath", sourcePath,
"--outputPath", workingDir.toString() + "/remapDoi"
});
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
JavaRDD<BipScore> tmp = sc
.textFile(workingDir.toString() + "/remapDoi")
.map(item -> OBJECT_MAPPER.readValue(item, BipScore.class));
Assertions.assertEquals(86, tmp.count());
// tmp.foreach(v -> System.out.println(OBJECT_MAPPER.writeValueAsString(v)));
String doi1 = ID_PREFIX +
IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.0000/096020199389707"));
Assertions.assertEquals(1, tmp.filter(r -> r.getId().equals(doi1)).count());
Assertions.assertEquals(3, tmp.filter(r -> r.getId().equals(doi1)).collect().get(0).getScoreList().size());
Assertions
.assertEquals(
"6.34596412687e-09", tmp
.filter(r -> r.getId().equals(doi1))
.collect()
.get(0)
.getScoreList()
.stream()
.filter(sl -> sl.getId().equals("influence"))
.collect(Collectors.toList())
.get(0)
.getUnit()
.get(0)
.getValue());
Assertions
.assertEquals(
"0.641151896994", tmp
.filter(r -> r.getId().equals(doi1))
.collect()
.get(0)
.getScoreList()
.stream()
.filter(sl -> sl.getId().equals("popularity_alt"))
.collect(Collectors.toList())
.get(0)
.getUnit()
.get(0)
.getValue());
Assertions
.assertEquals(
"2.33375102921e-09", tmp
.filter(r -> r.getId().equals(doi1))
.collect()
.get(0)
.getScoreList()
.stream()
.filter(sl -> sl.getId().equals("popularity"))
.collect(Collectors.toList())
.get(0)
.getUnit()
.get(0)
.getValue());
}
@Test
void updateResult() throws Exception {
final String bipScorePath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/bip/preparedbip.json")
.getPath();
final String inputPath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/bip/publicationnomatch.json")
.getPath();
SparkUpdateBip
.main(
new String[] {
"--isSparkSessionManaged", Boolean.FALSE.toString(),
"--bipScorePath", bipScorePath,
"--inputPath", inputPath,
"--outputPath", workingDir.toString() + "/publication",
"--resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication"
});
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
JavaRDD<Publication> tmp = sc
.textFile(workingDir.toString() + "/publication/bip")
.map(item -> OBJECT_MAPPER.readValue(item, Publication.class));
Assertions.assertEquals(6, tmp.count());
Assertions.assertEquals(0, tmp.filter(r -> r.getMeasures() != null).count());
tmp.foreach(r -> Assertions.assertEquals("publication", r.getResulttype().getClassid()));
}
@Test
void updateResultMatchCheckMeasures() throws Exception {
final String bipScorePath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/bip/preparedbip.json")
.getPath();
final String inputPath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/bip/publicationmatch.json")
.getPath();
SparkUpdateBip
.main(
new String[] {
"--isSparkSessionManaged", Boolean.FALSE.toString(),
"--bipScorePath", bipScorePath,
"--inputPath", inputPath,
"--outputPath", workingDir.toString() + "/publication",
"--resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication"
});
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
JavaRDD<Publication> tmp = sc
.textFile(workingDir.toString() + "/publication/bip")
.map(item -> OBJECT_MAPPER.readValue(item, Publication.class));
Assertions.assertEquals(6, tmp.count());
Assertions.assertEquals(1, tmp.filter(r -> r.getMeasures() != null).count());
Assertions
.assertEquals(
1, tmp.filter(r -> r.getId().equals("50|doi_________b24ab3e127aa67e2a1017292988d571f")).count());
Assertions
.assertEquals(
1,
tmp
.filter(
r -> r.getId().equals("50|doi_________b24ab3e127aa67e2a1017292988d571f")
&& r.getMeasures() != null)
.count());
Assertions.assertEquals(3, tmp
.filter(r -> r.getId().equals("50|doi_________b24ab3e127aa67e2a1017292988d571f"))
.collect()
.get(0)
.getMeasures().size());
Assertions.assertEquals("5.91019644836e-09",
tmp.filter(r -> r.getId().equals("50|doi_________b24ab3e127aa67e2a1017292988d571f"))
.collect()
.get(0).getMeasures().stream().filter(m -> m.getId().equals("influence")).collect(Collectors.toList()).get(0).getUnit().get(0).getValue());
Assertions.assertEquals("0.0",
tmp.filter(r -> r.getId().equals("50|doi_________b24ab3e127aa67e2a1017292988d571f"))
.collect()
.get(0).getMeasures().stream().filter(m -> m.getId().equals("popularity_alt")).collect(Collectors.toList()).get(0).getUnit().get(0).getValue());
Assertions.assertEquals("9.88840807598e-09",
tmp.filter(r -> r.getId().equals("50|doi_________b24ab3e127aa67e2a1017292988d571f"))
.collect()
.get(0).getMeasures().stream().filter(m -> m.getId().equals("popularity")).collect(Collectors.toList()).get(0).getUnit().get(0).getValue());
tmp.foreach(r -> System.out.println(OBJECT_MAPPER.writeValueAsString(r)));
}
}

253
dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bypassactionset/FOSTest.java Normal file
View File

@ -0,0 +1,253 @@
package eu.dnetlib.dhp.bypassactionset;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import eu.dnetlib.dhp.PropagationConstant;
import eu.dnetlib.dhp.bypassactionset.fos.SparkUpdateFOS;
import eu.dnetlib.dhp.schema.oaf.Publication;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.neethi.Assertion;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.bypassactionset.fos.GetFOSData;
import eu.dnetlib.dhp.bypassactionset.fos.PrepareFOSSparkJob;
import eu.dnetlib.dhp.bypassactionset.model.FOSDataModel;
import eu.dnetlib.dhp.common.collection.CollectorException;
import eu.dnetlib.dhp.countrypropagation.CountryPropagationJobTest;
import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
public class FOSTest {
private static final Logger log = LoggerFactory.getLogger(FOSTest.class);
private static Path workingDir;
private static SparkSession spark;
private static LocalFileSystem fs;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String ID_PREFIX = "50|doi_________";
@BeforeAll
public static void beforeAll() throws IOException {
workingDir = Files.createTempDirectory(CountryPropagationJobTest.class.getSimpleName());
fs = FileSystem.getLocal(new Configuration());
log.info("using work dir {}", workingDir);
SparkConf conf = new SparkConf();
conf.setAppName(FOSTest.class.getSimpleName());
conf.setMaster("local[*]");
conf.set("spark.driver.host", "localhost");
conf.set("hive.metastore.local", "true");
conf.set("spark.ui.enabled", "false");
conf.set("spark.sql.warehouse.dir", workingDir.toString());
conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
spark = SparkSession
.builder()
.appName(CountryPropagationJobTest.class.getSimpleName())
.config(conf)
.getOrCreate();
}
@AfterAll
public static void afterAll() throws IOException {
FileUtils.deleteDirectory(workingDir.toFile());
spark.stop();
}
@Test
void getFOSFileTest() throws CollectorException, IOException, ClassNotFoundException {
final String sourcePath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/fos/h2020_fos_sbs.csv")
.getPath();
final String outputPath = workingDir.toString() + "/fos.json";
new GetFOSData()
.doRewrite(sourcePath, outputPath, "eu.dnetlib.dhp.bypassactionset.FOSDataModel", '\t', fs);
BufferedReader in = new BufferedReader(
new InputStreamReader(fs.open(new org.apache.hadoop.fs.Path(outputPath))));
String line;
int count = 0;
while ((line = in.readLine()) != null) {
FOSDataModel fos = new ObjectMapper().readValue(line, FOSDataModel.class);
System.out.println(new ObjectMapper().writeValueAsString(fos));
count += 1;
}
assertEquals(38, count);
}
@Test
void distributeDoiTest() throws Exception {
final String sourcePath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/fos/fos.json")
.getPath();
PrepareFOSSparkJob
.main(
new String[] {
"--isSparkSessionManaged", Boolean.FALSE.toString(),
"--sourcePath", sourcePath,
"-outputPath", workingDir.toString() + "/distribute"
});
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
JavaRDD<FOSDataModel> tmp = sc
.textFile(workingDir.toString() + "/distribute")
.map(item -> OBJECT_MAPPER.readValue(item, FOSDataModel.class));
String doi1 = ID_PREFIX +
IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.3390/s18072310"));
assertEquals(50, tmp.count());
assertEquals(1, tmp.filter(row -> row.getDoi().equals(doi1)).count());
assertEquals(
"engineering and technology", tmp.filter(r -> r.getDoi().equals(doi1)).collect().get(0).getLevel1());
assertEquals("nano-technology", tmp.filter(r -> r.getDoi().equals(doi1)).collect().get(0).getLevel2());
assertEquals(
"nanoscience & nanotechnology", tmp.filter(r -> r.getDoi().equals(doi1)).collect().get(0).getLevel3());
String doi = ID_PREFIX +
IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.1111/1365-2656.12831"));
assertEquals(1, tmp.filter(row -> row.getDoi().equals(doi)).count());
assertEquals("social sciences", tmp.filter(r -> r.getDoi().equals(doi)).collect().get(0).getLevel1());
assertEquals(
"psychology and cognitive sciences", tmp.filter(r -> r.getDoi().equals(doi)).collect().get(0).getLevel2());
assertEquals("NULL", tmp.filter(r -> r.getDoi().equals(doi)).collect().get(0).getLevel3());
}
@Test
void updateResult() throws Exception{
final String fosPath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/fos/fos_prepared.json")
.getPath();
final String inputPath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/bip/publicationnomatch.json")
.getPath();
SparkUpdateFOS
.main(
new String[] {
"--isSparkSessionManaged", Boolean.FALSE.toString(),
"--fosPath", fosPath,
"--inputPath", inputPath,
"--outputPath", workingDir.toString() + "/publication",
"--resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication"
});
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
JavaRDD<Publication> tmp = sc
.textFile(workingDir.toString() + "/publication")
.map(item -> OBJECT_MAPPER.readValue(item, Publication.class));
Assertions.assertEquals(6, tmp.count());
tmp.filter(r -> r.getSubject() != null).map(p -> p.getSubject())
.foreach(s -> s.stream().forEach(sbj -> Assertions.assertFalse("FOS".equals(sbj.getQualifier().getClassid()))));
}
@Test
void updateResultMatch() throws Exception{
final String fosPath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/fos/fos_prepared.json")
.getPath();
final String inputPath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/fos/publicationmatch.json")
.getPath();
SparkUpdateFOS
.main(
new String[] {
"--isSparkSessionManaged", Boolean.FALSE.toString(),
"--fosPath", fosPath,
"--inputPath", inputPath,
"--outputPath", workingDir.toString() + "/publication",
"--resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication"
});
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
JavaRDD<Publication> tmp = sc
.textFile(workingDir.toString() + "/publication")
.map(item -> OBJECT_MAPPER.readValue(item, Publication.class));
Assertions.assertEquals(6, tmp.count());
Assertions.assertEquals(3, tmp.filter(r -> r.getSubject() != null).map(p -> p.getSubject()).flatMap(v -> v.iterator())
.filter(sbj -> sbj.getQualifier().getClassid().equals("FOS")).collect().size());
List<StructuredProperty> sbjs = tmp.filter(r -> r.getId().equals("50|doi_________b24ab3e127aa67e2a1017292988d571f"))
.map(p -> p.getSubject()).collect().get(0);
Assertions.assertEquals(12, sbjs.size());
Stream<StructuredProperty> fosSubjs = sbjs.stream().filter(sbj -> sbj.getQualifier().getClassid().equals("FOS"));
Assertions.assertTrue(fosSubjs
.map(sbj -> sbj.getValue()).collect(Collectors.toList()).contains("engineering and technology"));
Assertions.assertTrue(fosSubjs
.map(sbj -> sbj.getValue()).collect(Collectors.toList()).contains("nano-technology"));
Assertions.assertTrue(fosSubjs
.map(sbj -> sbj.getValue()).collect(Collectors.toList()).contains("nanoscience & nanotechnology"));
fosSubjs.forEach(sbj -> Assertions.assertEquals("update", sbj.getDataInfo().getInferenceprovenance()) );
fosSubjs.forEach(sbj -> Assertions.assertEquals("subject:fos", sbj.getDataInfo().getProvenanceaction().getClassid()) );
fosSubjs.forEach(sbj -> Assertions.assertEquals("Inferred by OpenAIRE", sbj.getDataInfo().getProvenanceaction().getClassname() ));
fosSubjs.forEach(sbj -> Assertions.assertEquals("", sbj.getDataInfo().getTrust() ));
fosSubjs.forEach(sbj -> Assertions.assertEquals(false, sbj.getDataInfo().getDeletedbyinference() ));
fosSubjs.forEach(sbj -> Assertions.assertEquals(false, sbj.getDataInfo().getInvisible() ));
fosSubjs.forEach(sbj -> Assertions.assertEquals(true, sbj.getDataInfo().getInferred() ));
}
}

341
dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bypassactionset/GetFOSTest.java
View File

@ -1,341 +0,0 @@
package eu.dnetlib.dhp.bypassactionset;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.bypassactionset.fos.PrepareFOSSparkJob;
import eu.dnetlib.dhp.bypassactionset.fos.GetFOSData;
import eu.dnetlib.dhp.bypassactionset.model.FOSDataModel;
import eu.dnetlib.dhp.common.collection.CollectorException;
import eu.dnetlib.dhp.countrypropagation.CountryPropagationJobTest;
import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import java.nio.file.Path;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class GetFOSTest {
private static final Logger log = LoggerFactory.getLogger(GetFOSTest.class);
private static Path workingDir;
private static SparkSession spark;
private static LocalFileSystem fs;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String ID_PREFIX = "50|doi_________";
@BeforeAll
public static void beforeAll() throws IOException {
workingDir = Files.createTempDirectory(CountryPropagationJobTest.class.getSimpleName());
fs = FileSystem.getLocal(new Configuration());
log.info("using work dir {}", workingDir);
SparkConf conf = new SparkConf();
conf.setAppName(GetFOSTest.class.getSimpleName());
conf.setMaster("local[*]");
conf.set("spark.driver.host", "localhost");
conf.set("hive.metastore.local", "true");
conf.set("spark.ui.enabled", "false");
conf.set("spark.sql.warehouse.dir", workingDir.toString());
conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
spark = SparkSession
.builder()
.appName(CountryPropagationJobTest.class.getSimpleName())
.config(conf)
.getOrCreate();
}
@AfterAll
public static void afterAll() throws IOException {
FileUtils.deleteDirectory(workingDir.toFile());
spark.stop();
}
@Test
void getFOSFileTest() throws CollectorException, IOException, ClassNotFoundException {
final String sourcePath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/fos/h2020_fos_sbs.csv")
.getPath();
final String outputPath = workingDir.toString() + "/fos.json";
new GetFOSData()
.doRewrite(sourcePath, outputPath, "eu.dnetlib.dhp.bypassactionset.FOSDataModel", '\t',fs );
BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new org.apache.hadoop.fs.Path(outputPath))));
String line;
int count = 0;
while ((line = in.readLine()) != null) {
FOSDataModel fos = new ObjectMapper().readValue(line, FOSDataModel.class);
System.out.println(new ObjectMapper().writeValueAsString(fos));
count += 1;
}
assertEquals(38, count);
}
@Test
void distributeDoiTest() throws Exception{
final String sourcePath = getClass()
.getResource("/eu/dnetlib/dhp/bypassactionset/fos/fos.json")
.getPath();
PrepareFOSSparkJob
.main(
new String[] {
"--isSparkSessionManaged", Boolean.FALSE.toString(),
"--sourcePath", sourcePath,
"-outputPath", workingDir.toString() + "/distribute"
});
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
JavaRDD<FOSDataModel> tmp = sc
.textFile(workingDir.toString() + "/distribute")
.map(item -> OBJECT_MAPPER.readValue(item, FOSDataModel.class));
String doi1 = ID_PREFIX +
IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.3390/s18072310"));
assertEquals(50, tmp.count());
assertEquals(1, tmp.filter(row -> row.getDoi().equals(doi1)).count());
assertEquals("engineering and technology", tmp.filter(r -> r.getDoi().equals(doi1)).collect().get(0).getLevel1());
assertEquals("nano-technology", tmp.filter(r -> r.getDoi().equals(doi1)).collect().get(0).getLevel2());
assertEquals("nanoscience & nanotechnology", tmp.filter(r -> r.getDoi().equals(doi1)).collect().get(0).getLevel3());
String doi = ID_PREFIX +
IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.1111/1365-2656.12831"));
assertEquals(1, tmp.filter(row -> row.getDoi().equals(doi)).count());
assertEquals("social sciences", tmp.filter(r -> r.getDoi().equals(doi)).collect().get(0).getLevel1());
assertEquals("psychology and cognitive sciences", tmp.filter(r -> r.getDoi().equals(doi)).collect().get(0).getLevel2());
assertEquals("NULL", tmp.filter(r -> r.getDoi().equals(doi)).collect().get(0).getLevel3());
// {"doi":"10.1111/1365-2656.12831\u000210.17863/cam.24369","level1":"social sciences","level2":"psychology and cognitive sciences","level3":"NULL"}
}
/**
* @Test
* void testCountryPropagationSoftware() throws Exception {
* final String sourcePath = getClass()
* .getResource("/eu/dnetlib/dhp/countrypropagation/sample/software")
* .getPath();
* final String preparedInfoPath = getClass()
* .getResource("/eu/dnetlib/dhp/countrypropagation/preparedInfo")
* .getPath();
* SparkCountryPropagationJob
* .main(
* new String[] {
* "--isSparkSessionManaged", Boolean.FALSE.toString(),
* "--sourcePath", sourcePath,
* "-saveGraph", "true",
* "-resultTableName", Software.class.getCanonicalName(),
* "-outputPath", workingDir.toString() + "/software",
* "-preparedInfoPath", preparedInfoPath
* });
*
* final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
*
* JavaRDD<Software> tmp = sc
* .textFile(workingDir.toString() + "/software")
* .map(item -> OBJECT_MAPPER.readValue(item, Software.class));
*
* // tmp.map(s -> new Gson().toJson(s)).foreach(s -> System.out.println(s));
*
* Assertions.assertEquals(10, tmp.count());
*
* Dataset<Software> verificationDs = spark.createDataset(tmp.rdd(), Encoders.bean(Software.class));
*
* Assertions.assertEquals(6, verificationDs.filter("size(country) > 0").count());
* Assertions.assertEquals(3, verificationDs.filter("size(country) = 1").count());
* Assertions.assertEquals(3, verificationDs.filter("size(country) = 2").count());
* Assertions.assertEquals(0, verificationDs.filter("size(country) > 2").count());
*
* Dataset<String> countryExploded = verificationDs
* .flatMap(
* (FlatMapFunction<Software, Country>) row -> row.getCountry().iterator(), Encoders.bean(Country.class))
* .map((MapFunction<Country, String>) Qualifier::getClassid, Encoders.STRING());
*
* Assertions.assertEquals(9, countryExploded.count());
*
* Assertions.assertEquals(1, countryExploded.filter("value = 'FR'").count());
* Assertions.assertEquals(1, countryExploded.filter("value = 'TR'").count());
* Assertions.assertEquals(2, countryExploded.filter("value = 'IT'").count());
* Assertions.assertEquals(1, countryExploded.filter("value = 'US'").count());
* Assertions.assertEquals(1, countryExploded.filter("value = 'MX'").count());
* Assertions.assertEquals(1, countryExploded.filter("value = 'CH'").count());
* Assertions.assertEquals(2, countryExploded.filter("value = 'JP'").count());
*
* Dataset<Tuple2<String, String>> countryExplodedWithCountryclassid = verificationDs
* .flatMap((FlatMapFunction<Software, Tuple2<String, String>>) row -> {
* List<Tuple2<String, String>> prova = new ArrayList<>();
* List<Country> countryList = row.getCountry();
* countryList
* .forEach(
* c -> prova
* .add(
* new Tuple2<>(
* row.getId(), c.getClassid())));
* return prova.iterator();
* }, Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
*
* Assertions.assertEquals(9, countryExplodedWithCountryclassid.count());
*
* countryExplodedWithCountryclassid.show(false);
* Assertions
* .assertEquals(
* 1,
* countryExplodedWithCountryclassid
* .filter(
* "_1 = '50|od______1582::6e7a9b21a2feef45673890432af34244' and _2 = 'FR' ")
* .count());
* Assertions
* .assertEquals(
* 1,
* countryExplodedWithCountryclassid
* .filter(
* "_1 = '50|dedup_wf_001::40ea2f24181f6ae77b866ebcbffba523' and _2 = 'TR' ")
* .count());
* Assertions
* .assertEquals(
* 2,
* countryExplodedWithCountryclassid
* .filter(
* "_1 = '50|od______1106::2b7ca9726230be8e862be224fd463ac4' and (_2 = 'IT' or _2 = 'MX') ")
* .count());
* Assertions
* .assertEquals(
* 2,
* countryExplodedWithCountryclassid
* .filter(
* "_1 = '50|od_______935::46a0ad9964171c3dd13373f5427b9a1c' and (_2 = 'IT' or _2 = 'US') ")
* .count());
* Assertions
* .assertEquals(
* 1,
* countryExplodedWithCountryclassid
* .filter(
* "_1 = '50|dedup_wf_001::b67bc915603fc01e445f2b5888ba7218' and _2 = 'JP'")
* .count());
* Assertions
* .assertEquals(
* 2,
* countryExplodedWithCountryclassid
* .filter(
* "_1 = '50|od_______109::f375befa62a741e9250e55bcfa88f9a6' and (_2 = 'CH' or _2 = 'JP') ")
* .count());
*
* Dataset<Tuple2<String, String>> countryExplodedWithCountryclassname = verificationDs
* .flatMap(
* (FlatMapFunction<Software, Tuple2<String, String>>) row -> {
* List<Tuple2<String, String>> prova = new ArrayList<>();
* List<Country> countryList = row.getCountry();
* countryList
* .forEach(
* c -> prova
* .add(
* new Tuple2<>(
* row.getId(),
* c.getClassname())));
* return prova.iterator();
* },
* Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
*
* countryExplodedWithCountryclassname.show(false);
* Assertions
* .assertEquals(
* 1,
* countryExplodedWithCountryclassname
* .filter(
* "_1 = '50|od______1582::6e7a9b21a2feef45673890432af34244' and _2 = 'France' ")
* .count());
* Assertions
* .assertEquals(
* 1,
* countryExplodedWithCountryclassname
* .filter(
* "_1 = '50|dedup_wf_001::40ea2f24181f6ae77b866ebcbffba523' and _2 = 'Turkey' ")
* .count());
* Assertions
* .assertEquals(
* 2,
* countryExplodedWithCountryclassname
* .filter(
* "_1 = '50|od______1106::2b7ca9726230be8e862be224fd463ac4' and (_2 = 'Italy' or _2 = 'Mexico') ")
* .count());
* Assertions
* .assertEquals(
* 2,
* countryExplodedWithCountryclassname
* .filter(
* "_1 = '50|od_______935::46a0ad9964171c3dd13373f5427b9a1c' and (_2 = 'Italy' or _2 = 'United States') ")
* .count());
* Assertions
* .assertEquals(
* 1,
* countryExplodedWithCountryclassname
* .filter(
* "_1 = '50|dedup_wf_001::b67bc915603fc01e445f2b5888ba7218' and _2 = 'Japan' ")
* .count());
* Assertions
* .assertEquals(
* 2,
* countryExplodedWithCountryclassname
* .filter(
* "_1 = '50|od_______109::f375befa62a741e9250e55bcfa88f9a6' and (_2 = 'Switzerland' or _2 = 'Japan') ")
* .count());
*
* Dataset<Tuple2<String, String>> countryExplodedWithCountryProvenance = verificationDs
* .flatMap(
* (FlatMapFunction<Software, Tuple2<String, String>>) row -> {
* List<Tuple2<String, String>> prova = new ArrayList<>();
* List<Country> countryList = row.getCountry();
* countryList
* .forEach(
* c -> prova
* .add(
* new Tuple2<>(
* row.getId(),
* c
* .getDataInfo()
* .getInferenceprovenance())));
* return prova.iterator();
* },
* Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
*
* Assertions
* .assertEquals(
* 7, countryExplodedWithCountryProvenance.filter("_2 = 'propagation'").count());
* }
*/
}

86
dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bypassactionset/bip/bip.json Normal file
View File

@ -0,0 +1,86 @@
{"10.0000/000000": [{"id": "influence", "unit": [{"value": "7.5597134689e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "4.903880192", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "1.17977512835e-08", "key": "score"}]}]}
{"10.0000/096020199389707": [{"id": "influence", "unit": [{"value": "6.34596412687e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.641151896994", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "2.33375102921e-09", "key": "score"}]}]}
{"10.00000/jpmc.2017.106": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "5.39172290649e-09", "key": "score"}]}]}
{"10.0000/9781845416881": [{"id": "influence", "unit": [{"value": "5.96492048955e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "1.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "1.12641925838e-08", "key": "score"}]}]}
{"10.0000/anziamj.v0i0.266": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.76260934675e-10", "key": "score"}]}]}
{"10.0000/anziamj.v48i0.79": [{"id": "influence", "unit": [{"value": "6.93311506443e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.002176782336", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "1.7668105708e-09", "key": "score"}]}]}
{"10.0000/anziamj.v50i0.1472": [{"id": "influence", "unit": [{"value": "6.26777280882e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.406656", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.39745193285e-09", "key": "score"}]}]}
{"10.0000/cja5553": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/czastest.16": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/czastest.17": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.47956715615e-09", "key": "score"}]}]}
{"10.0000/czastest.18": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.47956715615e-09", "key": "score"}]}]}
{"10.0000/czastest.20": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/czastest.21": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.47956715615e-09", "key": "score"}]}]}
{"10.0000/czastest.28": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.47956715615e-09", "key": "score"}]}]}
{"10.0000/czastest.60": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
{"10.0000/czt.2019.1.2.15": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v4i02.36": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v4i02.37": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v4i02.38": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v5i01.32": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v6i01.24": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v6i01.27": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v6i02.41": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v6i02.44": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v7i01.40": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v7i01.42": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v7i01.47": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v7i01.51": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v7i01.52": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v7i02.86": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v7i02.88": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v7i02.91": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v8i01.129": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v8i01.180": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "5.39172290649e-09", "key": "score"}]}]}
{"10.0000/geoekonomi.v8i01.87": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
{"10.0000/hbv2004w010": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hbv2101w001": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
{"10.0000/hbv2101w002": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
{"10.0000/hbv2101w003": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
{"10.0000/hbv2101w004": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
{"10.0000/hbv2101w005": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
{"10.0000/hbv2101w006": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
{"10.0000/hbv2101w007": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
{"10.0000/hbv2102w001": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
{"10.0000/hbv2102w010": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
{"10.0000/hoplos.v1i1.13207": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v1i1.13208": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "5.39172290649e-09", "key": "score"}]}]}
{"10.0000/hoplos.v1i1.13209": [{"id": "influence", "unit": [{"value": "6.32078461509e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "1.6", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.3168486939e-09", "key": "score"}]}]}
{"10.0000/hoplos.v1i1.13210": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v1i1.13211": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "5.39172290649e-09", "key": "score"}]}]}
{"10.0000/hoplos.v1i1.13212": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "5.39172290649e-09", "key": "score"}]}]}
{"10.0000/hoplos.v1i2.13231": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i2.28782": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i2.28783": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i2.28784": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i2.28786": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i2.28787": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i2.28788": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i3.28234": [{"id": "influence", "unit": [{"value": "6.40470414877e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.6", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.89465099068e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i3.28236": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i3.28238": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i3.28239": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i3.28242": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v2i3.28243": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
{"10.0000/hoplos.v3i4.38186": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
{"10.0000/hoplos.v3i4.38187": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
{"10.0000/hoplos.v3i4.38190": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
{"10.0000/hoplos.v3i4.38207": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
{"10.0000/hoplos.v3i4.38209": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
{"10.0000/hoplos.v3i5.41163": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
{"10.0000/hoplos.v3i5.41166": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
{"10.0000/hoplos.v3i5.41167": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
{"10.0000/hoplos.v3i5.41168": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
{"10.0000/hoplos.v3i5.41229": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i6.36360": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i6.40796": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i6.41153": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i6.42511": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i6.42555": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i6.42752": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i6.42768": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i6.42795": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i7.41295": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i7.42830": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i7.42861": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
{"10.0000/hoplos.v4i7.43096": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}

86
dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bypassactionset/bip/preparedbip.json Normal file
View File

@ -0,0 +1,86 @@
{"id":"50|doi_________63848be3afd635374828253a6f974f11","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"9.88840807598e-09"}]}]}
{"id":"50|doi_________5da9060b89165e3f61a0806dcf2c2696","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________9ea0266b8ddd471eddb20635b89273bb","scoreList":[{"id":"influence","unit":[{"key":"score","value":"7.5597134689e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"4.903880192"}]},{"id":"popularity","unit":[{"key":"score","value":"1.17977512835e-08"}]}]}
{"id":"50|doi_________ab797495de07d4f4a25f08b84fca6021","scoreList":[{"id":"influence","unit":[{"key":"score","value":"6.34596412687e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.641151896994"}]},{"id":"popularity","unit":[{"key":"score","value":"2.33375102921e-09"}]}]}
{"id":"50|doi_________df5ade937e177dcfb82bc5ec3139fefe","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"5.39172290649e-09"}]}]}
{"id":"50|doi_________357976ea7d4e744fe21966607334952c","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"5.39172290649e-09"}]}]}
{"id":"50|doi_________872982548f741b89eb5d30f509316dde","scoreList":[{"id":"influence","unit":[{"key":"score","value":"6.32078461509e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"1.6"}]},{"id":"popularity","unit":[{"key":"score","value":"8.3168486939e-09"}]}]}
{"id":"50|doi_________19c6da2771befb83f9f2715fc84f9edf","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.96492048955e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"1.0"}]},{"id":"popularity","unit":[{"key":"score","value":"1.12641925838e-08"}]}]}
{"id":"50|doi_________46e18e9e477d6fc71e002eae47cfc390","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________5cc344a6da53a8f2bac11faf7607e0b0","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"3.76260934675e-10"}]}]}
{"id":"50|doi_________fdfadf5cefdb8b63f90d5a3475a95959","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"5.39172290649e-09"}]}]}
{"id":"50|doi_________51fc7a9c7b9f1cf6705c7afb1de58967","scoreList":[{"id":"influence","unit":[{"key":"score","value":"6.93311506443e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.002176782336"}]},{"id":"popularity","unit":[{"key":"score","value":"1.7668105708e-09"}]}]}
{"id":"50|doi_________bbf7a94696ab67e4c29429522c31c0ef","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"5.39172290649e-09"}]}]}
{"id":"50|doi_________232498b3aed64dd0f0db87b09b61d5cd","scoreList":[{"id":"influence","unit":[{"key":"score","value":"6.26777280882e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.406656"}]},{"id":"popularity","unit":[{"key":"score","value":"3.39745193285e-09"}]}]}
{"id":"50|doi_________03749bed57efa24ab607527cf1eb94c5","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________65074998446928c1c18e25313dcf974b","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________2b62feaed6ad14760096c2a59f82836e","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________d2d133a6fdfd44f34f96e225b5573447","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________3dd3cccbfcad3f206d1239a580af011f","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________84852a20e9bbf1daf89803b5fcfc9c34","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"3.47956715615e-09"}]}]}
{"id":"50|doi_________84c5824b3a10a28a8bc26fed6223a08a","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________8658fda7574016daa0d71c88eb6bab5f","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"3.47956715615e-09"}]}]}
{"id":"50|doi_________ab52c136b88151f0c28f6ca4a5ef2a71","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________e8a2912fe3d70b13124e436294411378","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________cf3470c86f338ccf232f2869d0fa6ea2","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________16437c0064576207ba1438bf07fb9e21","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________f0ae1dc0f1dbb7fceaccc02d6e667f24","scoreList":[{"id":"influence","unit":[{"key":"score","value":"6.40470414877e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.6"}]},{"id":"popularity","unit":[{"key":"score","value":"7.89465099068e-09"}]}]}
{"id":"50|doi_________93d03d99bbc0f542fa2679e74882a096","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"3.47956715615e-09"}]}]}
{"id":"50|doi_________fcfce4bc985f22baf2dc1dc733a862e7","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________3b7fd390517f45b0d37e0ec0adbb98b1","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"3.47956715615e-09"}]}]}
{"id":"50|doi_________31306ba727c26ba04c0d4bbd0899d433","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________3e6151aa77865a1fdae56392f68b81f1","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.65008652949e-09"}]}]}
{"id":"50|doi_________1cd6e2c4cf189819fa8a011629b5744c","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"7.28336930301e-09"}]}]}
{"id":"50|doi_________45c855d81df747ff3c2033f6c8cf7bca","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________24ac4c0c4b143e37c2551723e9a55f36","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________5f9e3f1a076c2f587165f8ccd68286e5","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________b637731c913efe3c1f283183ef4a59cc","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"6.26204125721e-09"}]}]}
{"id":"50|doi_________490c0c23e4ed269f807e72e71cad09ae","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________15584e5a3a5bbdc487c85ab18d8a7c22","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"7.28336930301e-09"}]}]}
{"id":"50|doi_________a7fd268447553e7e0fe06c19db28ea85","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________cf1b8db4480aa0e281a4cc122d3b2416","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"7.28336930301e-09"}]}]}
{"id":"50|doi_________8880fd8fa9cafbde2431583dfbab1e1f","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________22f8b4f61c49de05dcc0d94bf5e147d8","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"7.28336930301e-09"}]}]}
{"id":"50|doi_________96e2b076fab4931ec2b1b9f43c9c31f9","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"7.28336930301e-09"}]}]}
{"id":"50|doi_________1ac665f912877f93f9084dc38bc8174b","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________394fd70c30b21b1336b5a0405496fbe0","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"7.28336930301e-09"}]}]}
{"id":"50|doi_________73caf44494091b089f8be65427fb4341","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________9b996963325dfae10dba5c3043938fc1","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"7.28336930301e-09"}]}]}
{"id":"50|doi_________1f03d5cab86f4963fbd3e0f1284cb9b8","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"7.28336930301e-09"}]}]}
{"id":"50|doi_________87a59f3918f9ac2c394b2509173fb3e1","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________9a8465f5343a1c845b6dd91496395c7e","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"7.28336930301e-09"}]}]}
{"id":"50|doi_________b03c0a657c669d24f45844307b9ad6bd","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________04aeabd709d8a486bde733ffc5ef53ae","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.01810569717e-09"}]}]}
{"id":"50|doi_________6c21d2d2133477141ad74226f8fd75b0","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"7.28336930301e-09"}]}]}
{"id":"50|doi_________b746ffb04cd9f815d5880c827c55040e","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________0457b2e62a69f0e104db7cc575f2241a","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.65008652949e-09"}]}]}
{"id":"50|doi_________2fa691f5394020f7cb2df4f272a062e7","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________aa2a77dc605567f5f2305b5f38a1a4b7","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.65008652949e-09"}]}]}
{"id":"50|doi_________cdb6f10588d85495208cbc6bf4a75000","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________6630766c2ad4783ebd01a9d7e9607eb1","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.65008652949e-09"}]}]}
{"id":"50|doi_________389315d5f74765688d34b8ee86c2514f","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________beef7316dd6665798120698451446d74","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.65008652949e-09"}]}]}
{"id":"50|doi_________525a46f117c946b8ae1caea542e24d0c","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________fe3d2250ebaa65dae09aa372945c917f","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.65008652949e-09"}]}]}
{"id":"50|doi_________72d2ec1744cda7d49cf35707ff3e11c1","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________d7e0c5fd1f00d8a90879f168d73b9e0b","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.65008652949e-09"}]}]}
{"id":"50|doi_________6e0e19ee1f9c1dbadf202ccd9919e6a7","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________6b306286f9b4d207f81bc9164713243c","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.65008652949e-09"}]}]}
{"id":"50|doi_________acf5c0c1a125cd78fd9193589edc9152","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________66f645f86cdd9592953a1106e0dbf191","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.65008652949e-09"}]}]}
{"id":"50|doi_________77ff842449308dd907ec46ebf9ccd539","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________a3d8fc73d1ac1308c117a61b4eb14ab5","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________9ff57cb17abfcd338c2ca768c85e452a","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"5.39172290649e-09"}]}]}
{"id":"50|doi_________f903c5c3305fc7b1598a0ed989bb4f83","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"4.65008652949e-09"}]}]}
{"id":"50|doi_________2260d2cdf6fbcee10d61d5e0f1428ebd","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________549c5c8f66c36cab609b7221eae37040","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________388a95b886946d771a7f38e86d58f65e","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________4141ef3f730fc811930a66c088486c34","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"9.88840807598e-09"}]}]}
{"id":"50|doi_________e3b32800bba3187fbf7f470df08755ea","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"8.48190886761e-09"}]}]}
{"id":"50|doi_________d8d02aa2c2347ecc628724265c5cae27","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"9.88840807598e-09"}]}]}
{"id":"50|doi_________08f1db95b06e62e34948f2b47ac449af","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"9.88840807598e-09"}]}]}
{"id":"50|doi_________cbbab3f7306b0573664b94615f5c04e2","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"9.88840807598e-09"}]}]}
{"id":"50|doi_________b24ab3e127aa67e2a1017292988d571f","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"9.88840807598e-09"}]}]}
{"id":"50|doi_________eb9a4f92bd6934727e2722d22340884a","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"9.88840807598e-09"}]}]}
{"id":"50|doi_________727484ef97b31ecb0806cc959869aa97","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"9.88840807598e-09"}]}]}
{"id":"50|doi_________224c865ef04d10c117913fa08eeb1be4","scoreList":[{"id":"influence","unit":[{"key":"score","value":"5.91019644836e-09"}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0"}]},{"id":"popularity","unit":[{"key":"score","value":"9.88840807598e-09"}]}]}

6
dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bypassactionset/bip/publicationmatch.json Normal file
View File

File diff suppressed because one or more lines are too long

6
dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bypassactionset/bip/publicationnomatch.json Normal file
View File

File diff suppressed because one or more lines are too long

50
dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bypassactionset/fos/fos_prepared.json Normal file
View File

@ -0,0 +1,50 @@
{"doi":"50|doi_________b24ab3e127aa67e2a1017292988d571f","level1":"engineering and technology","level2":"nano-technology","level3":"nanoscience & nanotechnology"}
{"doi":"50|doi_________f648499be6ba8e83226834167f22a7cb","level1":"social sciences","level2":"psychology and cognitive sciences","level3":"NULL"}
{"doi":"50|doi_________439b0885db30c66ecf62a2a6a4451116","level1":"social sciences","level2":"psychology and cognitive sciences","level3":"NULL"}
{"doi":"50|doi_________c22f065c9ab34fee87a3952fb79f5ee6","level1":"natural sciences","level2":"NULL","level3":"NULL"}
{"doi":"50|doi_________9ea3d8140aaa8add5e929be92f0dab13","level1":"natural sciences","level2":"NULL","level3":"NULL"}
{"doi":"50|doi_________12f446645a131ab55fc5dabf6692da31","level1":"medical and health sciences","level2":"clinical medicine","level3":"oncology & carcinogenesis"}
{"doi":"50|doi_________550a40c9b57fcc974c127ad69dd60df2","level1":"natural sciences","level2":"earth and related environmental sciences","level3":"environmental sciences"}
{"doi":"50|doi_________6b95eee8d0eff26b7cca8e960968ad62","level1":"medical and health sciences","level2":"clinical medicine","level3":"oncology & carcinogenesis"}
{"doi":"50|doi_________5ff24d570b3984ddcf04f58b771ad635","level1":"natural sciences","level2":"physical sciences","level3":"NULL"}
{"doi":"50|doi_________b909cd0953065bc611bac1d89e96bf6c","level1":"engineering and technology","level2":"other engineering and technologies","level3":"building & construction"}
{"doi":"50|doi_________d56d9dc21f317b3e009d5b6c8ea87212","level1":"natural sciences","level2":"physical sciences","level3":"nuclear & particles physics"}
{"doi":"50|doi_________3ea4d5309ce7934ed281342dd1f70867","level1":"natural sciences","level2":"NULL","level3":"NULL"}
{"doi":"50|doi_________f23029f6f070b4b2e1c852ce7f9f5f32","level1":"medical and health sciences","level2":"other medical science","level3":"health policy & services"}
{"doi":"50|doi_________2ea2433fb314647e72cab828dd529f00","level1":"natural sciences","level2":"biological sciences","level3":"plant biology & botany"}
{"doi":"50|doi_________ffe10c217b3a96f7584cf09fdad579e1","level1":"engineering and technology","level2":"NULL","level3":"NULL"}
{"doi":"50|doi_________45e682be5a57e2fabea2c02ba0752f1a","level1":"natural sciences","level2":"NULL","level3":"NULL"}
{"doi":"50|doi_________312f5db945545c66e6f18c62faf6290c","level1":"medical and health sciences","level2":"health sciences","level3":"NULL"}
{"doi":"50|doi_________d53bd3a4e48921415e554a4edc91d6db","level1":"natural sciences","level2":"physical sciences","level3":"NULL"}
{"doi":"50|doi_________2819592482582ff33363069d116abd64","level1":"natural sciences","level2":"physical sciences","level3":"nuclear & particles physics"}
{"doi":"50|doi_________e41a47550fed93904e443123b752bc2b","level1":"engineering and technology","level2":"electrical engineering, electronic engineering, information engineering","level3":"electrical & electronic engineering"}
{"doi":"50|doi_________43e8bf6e95cd3043f510771cf0b0c984","level1":"engineering and technology","level2":"mechanical engineering","level3":"mechanical engineering & transports"}
{"doi":"50|doi_________000c1dc14e99b89fc52976533338fe4c","level1":"engineering and technology","level2":"mechanical engineering","level3":"mechanical engineering & transports"}
{"doi":"50|doi_________5cf55e49aebd633f1326d28451d1f6e5","level1":"engineering and technology","level2":"electrical engineering, electronic engineering, information engineering","level3":"computer hardware & architecture"}
{"doi":"50|doi_________5b79bd7bd9f87361b4a4abc3cbb2df75","level1":"natural sciences","level2":"mathematics","level3":"numerical & computational mathematics"}
{"doi":"50|doi_________51a7b4738d332570beb98be13e95d369","level1":"natural sciences","level2":"chemical sciences","level3":"NULL"}
{"doi":"50|doi_________6a9271220296585204ff81d651215d63","level1":"medical and health sciences","level2":"basic medicine","level3":"NULL"}
{"doi":"50|doi_________a2b8554df106bb29a0035e2ea56046a2","level1":"medical and health sciences","level2":"health sciences","level3":"biochemistry & molecular biology"}
{"doi":"50|doi_________fa2b92d5140f67a6c69c970a9e8d2cf0","level1":"natural sciences","level2":"chemical sciences","level3":"physical chemistry"}
{"doi":"50|doi_________b5f6c78a31abbb806e1e375a73231dd1","level1":"natural sciences","level2":"chemical sciences","level3":"physical chemistry"}
{"doi":"50|doi_________358777f48b491554cdee63c4dcabe6aa","level1":"natural sciences","level2":"chemical sciences","level3":"physical chemistry"}
{"doi":"50|doi_________809f282159a9f46a41f92b6fc8d1095f","level1":"natural sciences","level2":"biological sciences","level3":"marine biology & hydrobiology"}
{"doi":"50|doi_________94f1b0b1509700d4370cbe7571000bfc","level1":"engineering and technology","level2":"industrial biotechnology","level3":"industrial engineering & automation"}
{"doi":"50|doi_________33390d9d5163da63f73acd173ef704e0","level1":"medical and health sciences","level2":"basic medicine","level3":"NULL"}
{"doi":"50|doi_________3149f0a909641720480f62faf1f886b9","level1":"medical and health sciences","level2":"basic medicine","level3":"NULL"}
{"doi":"50|doi_________1e29deb6473bdb2c84e5966fc4c557bb","level1":"medical and health sciences","level2":"basic medicine","level3":"NULL"}
{"doi":"50|doi_________effab60be43d653f2cccf1cf6de461df","level1":"engineering and technology","level2":"electrical engineering, electronic engineering, information engineering","level3":"electrical & electronic engineering"}
{"doi":"50|doi_________14a6ce8c28da6e82412720f1330e826d","level1":"natural sciences","level2":"biological sciences","level3":"NULL"}
{"doi":"50|doi_________f59abf7f96244398a465b6e9a7375311","level1":"natural sciences","level2":"earth and related environmental sciences","level3":"NULL"}
{"doi":"50|doi_________b1b7b5c7251fa1901583ec4840e2d3ec","level1":"medical and health sciences","level2":"basic medicine","level3":"biochemistry & molecular biology"}
{"doi":"50|doi_________e024d1b738df3b24bc58fa0228542571","level1":"natural sciences","level2":"physical sciences","level3":"nuclear & particles physics"}
{"doi":"50|doi_________0e03d3592dca1baedfc74f1fbe0b9f22","level1":"natural sciences","level2":"NULL","level3":"NULL"}
{"doi":"50|doi_________0e1777e31f32984e5a261205be28da69","level1":"engineering and technology","level2":"electrical engineering, electronic engineering, information engineering","level3":"networking & telecommunications"}
{"doi":"50|doi_________7b464a5eb02a959aeedc7b051fe0f89f","level1":"engineering and technology","level2":"electrical engineering, electronic engineering, information engineering","level3":"networking & telecommunications"}
{"doi":"50|doi_________32fed3ec7c9ac157b73e050ff1c158d3","level1":"engineering and technology","level2":"electrical engineering, electronic engineering, information engineering","level3":"networking & telecommunications"}
{"doi":"50|doi_________8302f548fe724bc58e53e2bd329cb3c3","level1":"medical and health sciences","level2":"health sciences","level3":"genetics & heredity"}
{"doi":"50|doi_________44fbef7e57e4123ef44f10bb073f9ef8","level1":"medical and health sciences","level2":"health sciences","level3":"genetics & heredity"}
{"doi":"50|doi_________c8e7d24b1649024e85bcf9a4c520a65b","level1":"medical and health sciences","level2":"health sciences","level3":"genetics & heredity"}
{"doi":"50|doi_________877a3c4a72d2a6b4aa60a2da016237df","level1":"medical and health sciences","level2":"health sciences","level3":"genetics & heredity"}
{"doi":"50|doi_________7af7cda00a082fa8624493d74357d3f7","level1":"engineering and technology","level2":"other engineering and technologies","level3":"building & construction"}
{"doi":"50|doi_________20201cc71fd003dae0d58016dd4ef4af","level1":"agricultural and veterinary sciences","level2":"agriculture, forestry, and fisheries","level3":"agronomy & agriculture"}

6
dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bypassactionset/fos/publicationmatch.json Normal file
View File

File diff suppressed because one or more lines are too long

2
pom.xml
View File

@ -753,7 +753,7 @@
<mockito-core.version>3.3.3</mockito-core.version> <mockito-core.version>3.3.3</mockito-core.version>
<mongodb.driver.version>3.4.2</mongodb.driver.version> <mongodb.driver.version>3.4.2</mongodb.driver.version>
<vtd.version>[2.12,3.0)</vtd.version> <vtd.version>[2.12,3.0)</vtd.version>
<dhp-schemas.version>[2.8.21]</dhp-schemas.version> <dhp-schemas.version>[2.8.22-SNAPSHOT]</dhp-schemas.version>
<dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version> <dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version>
<dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version> <dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version>
<dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version> <dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version>