forked from D-Net/dnet-hadoop
Added index wf
parent
2b8675462f
commit
b021b8a2e1
@ -0,0 +1,45 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>dhp-workflows</artifactId>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<version>1.0.5-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>dhp-graph-provision</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-core_2.11</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-sql_2.11</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-schemas</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch</groupId>
|
||||
<artifactId>elasticsearch-hadoop</artifactId>
|
||||
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
|
||||
</project>
|
||||
@ -0,0 +1,47 @@
|
||||
package eu.dnetlib.dhp.provision;
|
||||
|
||||
import eu.dnetlib.dhp.provision.scholix.Typology;
|
||||
import eu.dnetlib.dhp.utils.DHPUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
public class ProvisionUtil {
|
||||
|
||||
public final static String deletedByInferenceJPATH = "$.dataInfo.deletedbyinference";
|
||||
public final static String TARGETJSONPATH = "$.target";
|
||||
public final static String SOURCEJSONPATH = "$.source";
|
||||
|
||||
public static RelatedItemInfo getItemType(final String item, final String idPath) {
|
||||
String targetId = DHPUtils.getJPathString(idPath, item);
|
||||
switch (StringUtils.substringBefore(targetId, "|")) {
|
||||
case "50":
|
||||
return new RelatedItemInfo().setRelatedPublication(1);
|
||||
case "60":
|
||||
return new RelatedItemInfo().setRelatedDataset(1);
|
||||
case "70":
|
||||
return new RelatedItemInfo().setRelatedUnknown(1);
|
||||
default:
|
||||
throw new RuntimeException("Unknonw target ID");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static Boolean isNotDeleted(final String item) {
|
||||
return !"true".equalsIgnoreCase(DHPUtils.getJPathString(deletedByInferenceJPATH, item));
|
||||
}
|
||||
|
||||
public static Typology getItemTypeFromId(String id) {
|
||||
|
||||
switch (StringUtils.substringBefore(id, "|")) {
|
||||
case "50":
|
||||
return Typology.publication;
|
||||
case "60":
|
||||
return Typology.dataset;
|
||||
case "70":
|
||||
return Typology.unknown;
|
||||
default:
|
||||
throw new RuntimeException("Unknonw ID type");
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,64 @@
|
||||
package eu.dnetlib.dhp.provision;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* This class models the information of related items
|
||||
*/
|
||||
|
||||
public class RelatedItemInfo implements Serializable {
|
||||
|
||||
private String id;
|
||||
|
||||
private int relatedDataset = 0;
|
||||
|
||||
private int relatedPublication = 0;
|
||||
|
||||
private int relatedUnknown = 0;
|
||||
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public RelatedItemInfo setId(String id) {
|
||||
this.id = id;
|
||||
return this;
|
||||
}
|
||||
|
||||
public RelatedItemInfo add(RelatedItemInfo other) {
|
||||
if (other != null) {
|
||||
relatedDataset += other.getRelatedDataset();
|
||||
relatedPublication += other.getRelatedPublication();
|
||||
relatedUnknown += other.getRelatedUnknown();
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getRelatedDataset() {
|
||||
return relatedDataset;
|
||||
}
|
||||
|
||||
public RelatedItemInfo setRelatedDataset(int relatedDataset) {
|
||||
this.relatedDataset = relatedDataset;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getRelatedPublication() {
|
||||
return relatedPublication;
|
||||
}
|
||||
|
||||
public RelatedItemInfo setRelatedPublication(int relatedPublication) {
|
||||
this.relatedPublication = relatedPublication;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getRelatedUnknown() {
|
||||
return relatedUnknown;
|
||||
}
|
||||
|
||||
public RelatedItemInfo setRelatedUnknown(int relatedUnknown) {
|
||||
this.relatedUnknown = relatedUnknown;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,74 @@
|
||||
package eu.dnetlib.dhp.provision;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.jayway.jsonpath.JsonPath;
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
import eu.dnetlib.dhp.utils.DHPUtils;
|
||||
import net.minidev.json.JSONArray;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.io.compress.GzipCodec;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
import org.apache.spark.api.java.function.Function2;
|
||||
import org.apache.spark.api.java.function.PairFunction;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
import scala.Tuple2;
|
||||
|
||||
|
||||
/**
|
||||
* SparkExtractRelationCount is a spark job that takes in input relation RDD
|
||||
* and retrieve for each item in relation which are the number of
|
||||
* - Related Dataset
|
||||
* - Related Publication
|
||||
* - Related Unknown
|
||||
*/
|
||||
public class SparkExtractRelationCount {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(IOUtils.toString(SparkExtractRelationCount.class.getResourceAsStream("/eu/dnetlib/dhp/provision/input_related_entities_parameters.json")));
|
||||
parser.parseArgument(args);
|
||||
final SparkSession spark = SparkSession
|
||||
.builder()
|
||||
.appName(SparkExtractRelationCount.class.getSimpleName())
|
||||
.master(parser.get("master"))
|
||||
.getOrCreate();
|
||||
|
||||
|
||||
final String workingDirPath = parser.get("workingDirPath");
|
||||
|
||||
final String relationPath = parser.get("relationPath");
|
||||
|
||||
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
|
||||
|
||||
sc.textFile(relationPath)
|
||||
// We start to Filter the relation not deleted by Inference
|
||||
.filter(ProvisionUtil::isNotDeleted)
|
||||
// Then we create a PairRDD<String, RelatedItem>
|
||||
.mapToPair((PairFunction<String, String, RelatedItemInfo>) f
|
||||
-> new Tuple2<>(DHPUtils.getJPathString(ProvisionUtil.SOURCEJSONPATH, f), ProvisionUtil.getItemType(f, ProvisionUtil.TARGETJSONPATH)))
|
||||
//We reduce and sum the number of Relations
|
||||
.reduceByKey((Function2<RelatedItemInfo, RelatedItemInfo, RelatedItemInfo>) (v1, v2) -> {
|
||||
if (v1 == null && v2 == null)
|
||||
return new RelatedItemInfo();
|
||||
return v1 != null ? v1.add(v2) : v2;
|
||||
})
|
||||
//Set the source Id in RelatedItem object
|
||||
.map(k -> k._2().setId(k._1()))
|
||||
// Convert to JSON and save as TextFile
|
||||
.map(k -> {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
return mapper.writeValueAsString(k);
|
||||
}).saveAsTextFile(workingDirPath + "/relatedItemCount", GzipCodec.class);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
package eu.dnetlib.dhp.provision;
|
||||
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
import eu.dnetlib.dhp.provision.scholix.ScholixSummary;
|
||||
import eu.dnetlib.dhp.utils.DHPUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.hadoop.io.compress.GzipCodec;
|
||||
import org.apache.spark.api.java.JavaPairRDD;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
import org.apache.spark.api.java.function.Function;
|
||||
import org.apache.spark.api.java.function.PairFunction;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
import scala.Tuple2;
|
||||
|
||||
public class SparkGenerateSummary {
|
||||
|
||||
private static final String jsonIDPath = "$.id";
|
||||
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(IOUtils.toString(SparkGenerateSummary.class.getResourceAsStream("/eu/dnetlib/dhp/provision/input_generate_summary_parameters.json")));
|
||||
parser.parseArgument(args);
|
||||
final SparkSession spark = SparkSession
|
||||
.builder()
|
||||
.appName(SparkExtractRelationCount.class.getSimpleName())
|
||||
.master(parser.get("master"))
|
||||
.getOrCreate();
|
||||
|
||||
|
||||
final String graphPath = parser.get("graphPath");
|
||||
final String workingDirPath = parser.get("workingDirPath");
|
||||
|
||||
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
|
||||
JavaPairRDD<String, String> relationCount = sc.textFile(workingDirPath+"/relatedItemCount").mapToPair((PairFunction<String, String, String>) i -> new Tuple2<>(DHPUtils.getJPathString(jsonIDPath, i), i));
|
||||
|
||||
JavaPairRDD<String, String> entities =
|
||||
sc.textFile(graphPath + "/publication")
|
||||
.filter(ProvisionUtil::isNotDeleted)
|
||||
.mapToPair((PairFunction<String, String, String>) i -> new Tuple2<>(DHPUtils.getJPathString(jsonIDPath, i), i))
|
||||
.union(
|
||||
sc.textFile(graphPath + "/dataset")
|
||||
.filter(ProvisionUtil::isNotDeleted)
|
||||
.mapToPair((PairFunction<String, String, String>) i -> new Tuple2<>(DHPUtils.getJPathString(jsonIDPath, i), i))
|
||||
)
|
||||
.union(
|
||||
sc.textFile(graphPath + "/unknown")
|
||||
.filter(ProvisionUtil::isNotDeleted)
|
||||
.mapToPair((PairFunction<String, String, String>) i -> new Tuple2<>(DHPUtils.getJPathString(jsonIDPath, i), i))
|
||||
);
|
||||
entities.join(relationCount).map((Function<Tuple2<String, Tuple2<String, String>>, String>) k ->
|
||||
ScholixSummary.fromJsonOAF(ProvisionUtil.getItemTypeFromId(k._1()), k._2()._1(), k._2()._2())).saveAsTextFile(workingDirPath+"/summary", GzipCodec.class);
|
||||
|
||||
|
||||
;
|
||||
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
package eu.dnetlib.dhp.provision;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
import eu.dnetlib.dhp.provision.scholix.ScholixSummary;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.JavaRDD;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
import org.elasticsearch.spark.rdd.api.java.JavaEsSpark;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class SparkIndexCollectionOnES {
|
||||
|
||||
public static void main(String[] args) throws Exception{
|
||||
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(IOUtils.toString(SparkIndexCollectionOnES.class.getResourceAsStream("/eu/dnetlib/dhp/provision/index_on_es.json")));
|
||||
parser.parseArgument(args);
|
||||
|
||||
SparkConf conf = new SparkConf().setAppName(SparkIndexCollectionOnES.class.getSimpleName())
|
||||
.setMaster(parser.get("master"));
|
||||
|
||||
|
||||
final String sourcePath = parser.get("sourcePath");
|
||||
final String index = parser.get("index");
|
||||
|
||||
final SparkSession spark = SparkSession.builder().config(conf).getOrCreate();
|
||||
|
||||
|
||||
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
|
||||
|
||||
JavaRDD<String> inputRdd = sc.textFile(sourcePath);
|
||||
|
||||
Map<String, String> esCfg = new HashMap<>();
|
||||
esCfg.put("es.nodes", "10.19.65.51, 10.19.65.52, 10.19.65.53, 10.19.65.54");
|
||||
esCfg.put("es.mapping.id", "id");
|
||||
esCfg.put("es.batch.write.retry.count", "8");
|
||||
esCfg.put("es.batch.write.retry.wait", "60s");
|
||||
esCfg.put("es.batch.size.entries", "200");
|
||||
esCfg.put("es.nodes.wan.only", "true");
|
||||
|
||||
|
||||
JavaEsSpark.saveJsonToEs(inputRdd,index, esCfg);
|
||||
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
package eu.dnetlib.dhp.provision.scholix;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
public class CollectedFromType implements Serializable {
|
||||
|
||||
private String datasourceName;
|
||||
private String datasourceId;
|
||||
private String completionStatus;
|
||||
|
||||
|
||||
public CollectedFromType() {
|
||||
}
|
||||
|
||||
public CollectedFromType(String datasourceName, String datasourceId, String completionStatus) {
|
||||
this.datasourceName = datasourceName;
|
||||
this.datasourceId = datasourceId;
|
||||
this.completionStatus = completionStatus;
|
||||
}
|
||||
|
||||
public String getDatasourceName() {
|
||||
return datasourceName;
|
||||
}
|
||||
|
||||
public void setDatasourceName(String datasourceName) {
|
||||
this.datasourceName = datasourceName;
|
||||
}
|
||||
|
||||
public String getDatasourceId() {
|
||||
return datasourceId;
|
||||
}
|
||||
|
||||
public void setDatasourceId(String datasourceId) {
|
||||
this.datasourceId = datasourceId;
|
||||
}
|
||||
|
||||
public String getCompletionStatus() {
|
||||
return completionStatus;
|
||||
}
|
||||
|
||||
public void setCompletionStatus(String completionStatus) {
|
||||
this.completionStatus = completionStatus;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
package eu.dnetlib.dhp.provision.scholix;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
public class SchemeValue implements Serializable {
|
||||
private String scheme;
|
||||
private String value;
|
||||
|
||||
public SchemeValue() {
|
||||
|
||||
}
|
||||
|
||||
public SchemeValue(String scheme, String value) {
|
||||
this.scheme = scheme;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public String getScheme() {
|
||||
return scheme;
|
||||
}
|
||||
|
||||
public void setScheme(String scheme) {
|
||||
this.scheme = scheme;
|
||||
}
|
||||
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public void setValue(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,289 @@
|
||||
package eu.dnetlib.dhp.provision.scholix;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import eu.dnetlib.dhp.provision.RelatedItemInfo;
|
||||
import eu.dnetlib.dhp.schema.oaf.Author;
|
||||
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
|
||||
import eu.dnetlib.dhp.schema.scholexplorer.DLIDataset;
|
||||
import eu.dnetlib.dhp.schema.scholexplorer.DLIPublication;
|
||||
import eu.dnetlib.dhp.schema.scholexplorer.DLIUnknown;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ScholixSummary implements Serializable {
|
||||
private String id;
|
||||
private List<TypedIdentifier> localIdentifier;
|
||||
private Typology typology;
|
||||
private List<String> title;
|
||||
private List<String> author;
|
||||
private List<String> date;
|
||||
private String description;
|
||||
private List<SchemeValue> subject;
|
||||
private List<String> publisher;
|
||||
private int relatedPublications;
|
||||
private int relatedDatasets;
|
||||
private int relatedUnknown;
|
||||
private List<CollectedFromType> datasources;
|
||||
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public List<TypedIdentifier> getLocalIdentifier() {
|
||||
return localIdentifier;
|
||||
}
|
||||
|
||||
public void setLocalIdentifier(List<TypedIdentifier> localIdentifier) {
|
||||
this.localIdentifier = localIdentifier;
|
||||
}
|
||||
|
||||
public Typology getTypology() {
|
||||
return typology;
|
||||
}
|
||||
|
||||
public void setTypology(Typology typology) {
|
||||
this.typology = typology;
|
||||
}
|
||||
|
||||
public List<String> getTitle() {
|
||||
return title;
|
||||
}
|
||||
|
||||
public void setTitle(List<String> title) {
|
||||
this.title = title;
|
||||
}
|
||||
|
||||
public List<String> getAuthor() {
|
||||
return author;
|
||||
}
|
||||
|
||||
public void setAuthor(List<String> author) {
|
||||
this.author = author;
|
||||
}
|
||||
|
||||
public List<String> getDate() {
|
||||
return date;
|
||||
}
|
||||
|
||||
public void setDate(List<String> date) {
|
||||
this.date = date;
|
||||
}
|
||||
|
||||
@JsonProperty("abstract")
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
|
||||
@JsonProperty("abstract")
|
||||
public void setDescription(String description) {
|
||||
this.description = description;
|
||||
}
|
||||
|
||||
public List<SchemeValue> getSubject() {
|
||||
return subject;
|
||||
}
|
||||
|
||||
public void setSubject(List<SchemeValue> subject) {
|
||||
this.subject = subject;
|
||||
}
|
||||
|
||||
public List<String> getPublisher() {
|
||||
return publisher;
|
||||
}
|
||||
|
||||
public void setPublisher(List<String> publisher) {
|
||||
this.publisher = publisher;
|
||||
}
|
||||
|
||||
public int getRelatedPublications() {
|
||||
return relatedPublications;
|
||||
}
|
||||
|
||||
public void setRelatedPublications(int relatedPublications) {
|
||||
this.relatedPublications = relatedPublications;
|
||||
}
|
||||
|
||||
public int getRelatedDatasets() {
|
||||
return relatedDatasets;
|
||||
}
|
||||
|
||||
public void setRelatedDatasets(int relatedDatasets) {
|
||||
this.relatedDatasets = relatedDatasets;
|
||||
}
|
||||
|
||||
public int getRelatedUnknown() {
|
||||
return relatedUnknown;
|
||||
}
|
||||
|
||||
public void setRelatedUnknown(int relatedUnknown) {
|
||||
this.relatedUnknown = relatedUnknown;
|
||||
}
|
||||
|
||||
public List<CollectedFromType> getDatasources() {
|
||||
return datasources;
|
||||
}
|
||||
|
||||
public void setDatasources(List<CollectedFromType> datasources) {
|
||||
this.datasources = datasources;
|
||||
}
|
||||
|
||||
|
||||
public static String fromJsonOAF(final Typology oafType, final String oafJson, final String relEntityJson) {
|
||||
try {
|
||||
final ObjectMapper mapper = new ObjectMapper();
|
||||
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
|
||||
RelatedItemInfo relatedItemInfo = mapper.readValue(relEntityJson, RelatedItemInfo.class);
|
||||
|
||||
switch (oafType) {
|
||||
case dataset:
|
||||
return mapper.writeValueAsString(summaryFromDataset(mapper.readValue(oafJson, DLIDataset.class), relatedItemInfo));
|
||||
case publication:
|
||||
return mapper.writeValueAsString(summaryFromPublication(mapper.readValue(oafJson, DLIPublication.class), relatedItemInfo));
|
||||
case unknown:
|
||||
return mapper.writeValueAsString(summaryFromUnknown(mapper.readValue(oafJson, DLIUnknown.class), relatedItemInfo));
|
||||
}
|
||||
|
||||
|
||||
} catch (Throwable e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
private static ScholixSummary summaryFromDataset(final DLIDataset item, final RelatedItemInfo relatedItemInfo) {
|
||||
ScholixSummary summary = new ScholixSummary();
|
||||
summary.setId(item.getId());
|
||||
|
||||
if (item.getPid() != null)
|
||||
summary.setLocalIdentifier(item.getPid().stream()
|
||||
.map(p -> new TypedIdentifier(p.getValue(), p.getQualifier().getClassid()))
|
||||
.collect(Collectors.toList())
|
||||
);
|
||||
|
||||
summary.setTypology(Typology.dataset);
|
||||
if (item.getTitle() != null)
|
||||
summary.setTitle(item.getTitle().stream().map(StructuredProperty::getValue).collect(Collectors.toList()));
|
||||
|
||||
if (item.getAuthor() != null) {
|
||||
summary.setAuthor(item.getAuthor().stream().map(Author::getFullname).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
if (item.getRelevantdate() != null)
|
||||
summary.setDate(
|
||||
item.getRelevantdate().stream()
|
||||
.filter(d -> "date".equalsIgnoreCase(d.getQualifier().getClassname()))
|
||||
.map(StructuredProperty::getValue)
|
||||
.collect(Collectors.toList())
|
||||
);
|
||||
|
||||
if (item.getDescription() != null && item.getDescription().size() > 0)
|
||||
summary.setDescription(item.getDescription().get(0).getValue());
|
||||
|
||||
if (item.getSubject() != null) {
|
||||
summary.setSubject(item.getSubject().stream()
|
||||
.map(s -> new SchemeValue(s.getQualifier().getClassid(), s.getValue()))
|
||||
.collect(Collectors.toList())
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
summary.setRelatedDatasets(relatedItemInfo.getRelatedDataset());
|
||||
summary.setRelatedPublications(relatedItemInfo.getRelatedPublication());
|
||||
summary.setRelatedUnknown(relatedItemInfo.getRelatedUnknown());
|
||||
|
||||
if (item.getDlicollectedfrom() != null)
|
||||
summary.setDatasources(item.getDlicollectedfrom().stream()
|
||||
.map(
|
||||
c -> new CollectedFromType(c.getName(), c.getId(), c.getCompletionStatus())
|
||||
).collect(Collectors.toList()));
|
||||
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
private static ScholixSummary summaryFromPublication(final DLIPublication item, final RelatedItemInfo relatedItemInfo) {
|
||||
ScholixSummary summary = new ScholixSummary();
|
||||
summary.setId(item.getId());
|
||||
|
||||
if (item.getPid() != null)
|
||||
summary.setLocalIdentifier(item.getPid().stream()
|
||||
.map(p -> new TypedIdentifier(p.getValue(), p.getQualifier().getClassid()))
|
||||
.collect(Collectors.toList())
|
||||
);
|
||||
|
||||
summary.setTypology(Typology.dataset);
|
||||
if (item.getTitle() != null)
|
||||
summary.setTitle(item.getTitle().stream().map(StructuredProperty::getValue).collect(Collectors.toList()));
|
||||
|
||||
if (item.getAuthor() != null) {
|
||||
summary.setAuthor(item.getAuthor().stream().map(Author::getFullname).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
if (item.getRelevantdate() != null)
|
||||
summary.setDate(
|
||||
item.getRelevantdate().stream()
|
||||
.filter(d -> "date".equalsIgnoreCase(d.getQualifier().getClassname()))
|
||||
.map(StructuredProperty::getValue)
|
||||
.collect(Collectors.toList())
|
||||
);
|
||||
|
||||
if (item.getDescription() != null && item.getDescription().size() > 0)
|
||||
summary.setDescription(item.getDescription().get(0).getValue());
|
||||
|
||||
if (item.getSubject() != null) {
|
||||
summary.setSubject(item.getSubject().stream()
|
||||
.map(s -> new SchemeValue(s.getQualifier().getClassid(), s.getValue()))
|
||||
.collect(Collectors.toList())
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
summary.setRelatedDatasets(relatedItemInfo.getRelatedDataset());
|
||||
summary.setRelatedPublications(relatedItemInfo.getRelatedPublication());
|
||||
summary.setRelatedUnknown(relatedItemInfo.getRelatedUnknown());
|
||||
|
||||
if (item.getDlicollectedfrom() != null)
|
||||
summary.setDatasources(item.getDlicollectedfrom().stream()
|
||||
.map(
|
||||
c -> new CollectedFromType(c.getName(), c.getId(), c.getCompletionStatus())
|
||||
).collect(Collectors.toList()));
|
||||
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
private static ScholixSummary summaryFromUnknown(final DLIUnknown item, final RelatedItemInfo relatedItemInfo) {
|
||||
ScholixSummary summary = new ScholixSummary();
|
||||
summary.setId(item.getId());
|
||||
if (item.getPid() != null)
|
||||
summary.setLocalIdentifier(item.getPid().stream()
|
||||
.map(p -> new TypedIdentifier(p.getValue(), p.getQualifier().getClassid()))
|
||||
.collect(Collectors.toList())
|
||||
);
|
||||
|
||||
summary.setRelatedDatasets(relatedItemInfo.getRelatedDataset());
|
||||
summary.setRelatedPublications(relatedItemInfo.getRelatedPublication());
|
||||
summary.setRelatedUnknown(relatedItemInfo.getRelatedUnknown());
|
||||
|
||||
if (item.getDlicollectedfrom() != null)
|
||||
summary.setDatasources(item.getDlicollectedfrom().stream()
|
||||
.map(
|
||||
c -> new CollectedFromType(c.getName(), c.getId(), c.getCompletionStatus())
|
||||
).collect(Collectors.toList()));
|
||||
|
||||
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
package eu.dnetlib.dhp.provision.scholix;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
public class TypedIdentifier implements Serializable {
|
||||
private String id;
|
||||
private String type;
|
||||
|
||||
public TypedIdentifier() {
|
||||
}
|
||||
|
||||
public TypedIdentifier(String id, String type) {
|
||||
this.id = id;
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package eu.dnetlib.dhp.provision.scholix;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
public enum Typology implements Serializable {
|
||||
dataset,
|
||||
publication,
|
||||
unknown
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
<configuration>
|
||||
<property>
|
||||
<name>oozie.use.system.libpath</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.action.sharelib.for.spark</name>
|
||||
<value>spark2</value>
|
||||
</property>
|
||||
</configuration>
|
||||
@ -0,0 +1,100 @@
|
||||
<workflow-app name="import_infospace_graph" xmlns="uri:oozie:workflow:0.5">
|
||||
<parameters>
|
||||
<property>
|
||||
<name>workingDirPath</name>
|
||||
<description>the source path</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>graphPath</name>
|
||||
<description>the graph path</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>index</name>
|
||||
<description>index name</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>sparkDriverMemory</name>
|
||||
<description>memory for driver process</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>sparkExecutorMemory</name>
|
||||
<description>memory for individual executor</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>sparkExecutorCores</name>
|
||||
<description>number of cores used by single executor</description>
|
||||
</property>
|
||||
|
||||
</parameters>
|
||||
|
||||
<start to="indexSummary"/>
|
||||
|
||||
<kill name="Kill">
|
||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||
</kill>
|
||||
<action name="DeleteTargetPath">
|
||||
<fs>
|
||||
<delete path='${workingDirPath}'/>
|
||||
<mkdir path='${workingDirPath}'/>
|
||||
</fs>
|
||||
<ok to="CalculateRelatedItem"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="CalculateRelatedItem">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>calculate for each ID the number of related Dataset, publication and Unknown</name>
|
||||
<class>eu.dnetlib.dhp.provision.SparkExtractRelationCount</class>
|
||||
<jar>dhp-graph-provision-${projectVersion}.jar</jar>
|
||||
<spark-opts>--executor-memory ${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} ${sparkExtraOPT}</spark-opts>
|
||||
<arg>-mt</arg> <arg>yarn-cluster</arg>
|
||||
<arg>--workingDirPath</arg><arg>${workingDirPath}</arg>
|
||||
<arg>--relationPath</arg><arg>${graphPath}/relation</arg>
|
||||
</spark>
|
||||
<ok to="generateSummary"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="generateSummary">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>generate Summary</name>
|
||||
<class>eu.dnetlib.dhp.provision.SparkGenerateSummary</class>
|
||||
<jar>dhp-graph-provision-${projectVersion}.jar</jar>
|
||||
<spark-opts>--executor-memory ${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} ${sparkExtraOPT}</spark-opts>
|
||||
<arg>-mt</arg> <arg>yarn-cluster</arg>
|
||||
<arg>--workingDirPath</arg><arg>${workingDirPath}</arg>
|
||||
<arg>--graphPath</arg><arg>${graphPath}</arg>
|
||||
</spark>
|
||||
<ok to="indexSummary"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="indexSummary">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>generate Summary</name>
|
||||
<class>eu.dnetlib.dhp.provision.SparkIndexCollectionOnES</class>
|
||||
<jar>dhp-graph-provision-${projectVersion}.jar</jar>
|
||||
<spark-opts>--executor-memory ${sparkExecutorMemory} --num-executors 20 --driver-memory=${sparkDriverMemory} ${sparkExtraOPT} </spark-opts>
|
||||
<arg>-mt</arg> <arg>yarn-cluster</arg>
|
||||
<arg>--sourcePath</arg><arg>${workingDirPath}/summary</arg>
|
||||
<arg>--index</arg><arg>${index}_object</arg>
|
||||
</spark>
|
||||
<ok to="End"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
<end name="End"/>
|
||||
|
||||
|
||||
</workflow-app>
|
||||
@ -0,0 +1,20 @@
|
||||
[
|
||||
{
|
||||
"paramName": "mt",
|
||||
"paramLongName": "master",
|
||||
"paramDescription": "should be local or yarn",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "s",
|
||||
"paramLongName": "sourcePath",
|
||||
"paramDescription": "the working path where generated files",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "i",
|
||||
"paramLongName": "index",
|
||||
"paramDescription": "the index name",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,20 @@
|
||||
[
|
||||
{
|
||||
"paramName": "mt",
|
||||
"paramLongName": "master",
|
||||
"paramDescription": "should be local or yarn",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "w",
|
||||
"paramLongName": "workingDirPath",
|
||||
"paramDescription": "the working path where generated files",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "g",
|
||||
"paramLongName": "graphPath",
|
||||
"paramDescription": "the relationPath path ",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,20 @@
|
||||
[
|
||||
{
|
||||
"paramName": "mt",
|
||||
"paramLongName": "master",
|
||||
"paramDescription": "should be local or yarn",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "w",
|
||||
"paramLongName": "workingDirPath",
|
||||
"paramDescription": "the working path where generated files",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "r",
|
||||
"paramLongName": "relationPath",
|
||||
"paramDescription": "the relationPath path ",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,48 @@
|
||||
package eu.dnetlib.dhp.provision;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
import eu.dnetlib.dhp.provision.scholix.ScholixSummary;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
public class ExtractInfoTest {
|
||||
|
||||
@Test
|
||||
public void test() throws Exception {
|
||||
|
||||
final String json = IOUtils.toString(getClass().getResourceAsStream("record.json"));
|
||||
|
||||
|
||||
ProvisionUtil.getItemType(json,ProvisionUtil.TARGETJSONPATH);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSerialization() throws Exception {
|
||||
|
||||
ScholixSummary summary = new ScholixSummary();
|
||||
summary.setDescription("descrizione");
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
String json = mapper.writeValueAsString(summary);
|
||||
System.out.println(json);
|
||||
System.out.println(mapper.readValue(json, ScholixSummary.class).getDescription());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@Ignore
|
||||
public void testIndex() throws Exception {
|
||||
SparkIndexCollectionOnES.main(
|
||||
|
||||
new String[] {
|
||||
"-mt", "local[*]",
|
||||
"-s", "/home/sandro/dli",
|
||||
"-i", "dli_object"
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
{"dataInfo":{"invisible":false,"inferred":null,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":null,"provenanceaction":null},"lastupdatetimestamp":null,"relType":"references","subRelType":null,"relClass":"datacite","source":"50|f2123fce7e56c73dc8f1bf64ec59b477","target":"50|b618cbe39ba940a29993ac324e5f9621","collectedFrom":[{"key":"dli_________::datacite","value":"Datasets in Datacite","dataInfo":null}]}
|
||||
Loading…
Reference in New Issue