antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' of code-repo.d4science.org:D-Net/dnet-hadoop

This commit is contained in:
Michele Artini 2020-05-05 17:12:22 +02:00
parent ccc609f909 0825321d0b
commit 8f30a09d84
22 changed files with 1673 additions and 415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java
View File

@ -7,6 +7,9 @@ import java.util.Objects;
public abstract class Oaf implements Serializable { public abstract class Oaf implements Serializable {
/**
* The list of datasource id/name pairs providing this relationship.
*/
protected List<KeyValue> collectedfrom; protected List<KeyValue> collectedfrom;
private DataInfo dataInfo; private DataInfo dataInfo;

22
dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java
View File

@ -7,16 +7,38 @@ import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
/**
* Relation models any edge between two nodes in the OpenAIRE graph. It has a source id and a target id pointing to
* graph node identifiers and it is further characterised by the semantic of the link through the fields relType,
* subRelType and relClass. Provenance information is modeled according to the dataInfo element and collectedFrom, while
* individual relationship types can provide extra information via the properties field.
*/
public class Relation extends Oaf { public class Relation extends Oaf {
/**
* Main relationship classifier, values include 'resultResult', 'resultProject', 'resultOrganization', etc.
*/
private String relType; private String relType;
/**
* Further classifies a relationship, values include 'affiliation', 'similarity', 'supplement', etc.
*/
private String subRelType; private String subRelType;
/**
* Indicates the direction of the relationship, values include 'isSupplementTo', 'isSupplementedBy', 'merges,
* 'isMergedIn'.
*/
private String relClass; private String relClass;
/**
* The source entity id.
*/
private String source; private String source;
/**
* The target entity id.
*/
private String target; private String target;
public String getRelType() { public String getRelType() {

146
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/GenerateNativeStoreSparkJob.java
View File

@ -1,17 +1,21 @@
package eu.dnetlib.dhp.collection; package eu.dnetlib.dhp.collection;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Optional;
import org.apache.commons.cli.*; import org.apache.commons.cli.*;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
@ -23,6 +27,8 @@ import org.apache.spark.util.LongAccumulator;
import org.dom4j.Document; import org.dom4j.Document;
import org.dom4j.Node; import org.dom4j.Node;
import org.dom4j.io.SAXReader; import org.dom4j.io.SAXReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
@ -35,6 +41,8 @@ import eu.dnetlib.message.MessageType;
public class GenerateNativeStoreSparkJob { public class GenerateNativeStoreSparkJob {
private static final Logger log = LoggerFactory.getLogger(GenerateNativeStoreSparkJob.class);
public static MetadataRecord parseRecord( public static MetadataRecord parseRecord(
final String input, final String input,
final String xpath, final String xpath,
@ -78,84 +86,90 @@ public class GenerateNativeStoreSparkJob {
final Provenance provenance = jsonMapper.readValue(parser.get("provenance"), Provenance.class); final Provenance provenance = jsonMapper.readValue(parser.get("provenance"), Provenance.class);
final long dateOfCollection = new Long(parser.get("dateOfCollection")); final long dateOfCollection = new Long(parser.get("dateOfCollection"));
final SparkSession spark = SparkSession Boolean isSparkSessionManaged = Optional
.builder() .ofNullable(parser.get("isSparkSessionManaged"))
.appName("GenerateNativeStoreSparkJob") .map(Boolean::valueOf)
.master(parser.get("master")) .orElse(Boolean.TRUE);
.getOrCreate(); log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
final Map<String, String> ongoingMap = new HashMap<>(); final Map<String, String> ongoingMap = new HashMap<>();
final Map<String, String> reportMap = new HashMap<>(); final Map<String, String> reportMap = new HashMap<>();
final boolean test = parser.get("isTest") == null ? false : Boolean.valueOf(parser.get("isTest")); final boolean test = parser.get("isTest") == null ? false : Boolean.valueOf(parser.get("isTest"));
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); SparkConf conf = new SparkConf();
runWithSparkSession(
conf,
isSparkSessionManaged,
spark -> {
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
final JavaPairRDD<IntWritable, Text> inputRDD = sc final JavaPairRDD<IntWritable, Text> inputRDD = sc
.sequenceFile(parser.get("input"), IntWritable.class, Text.class); .sequenceFile(parser.get("input"), IntWritable.class, Text.class);
final LongAccumulator totalItems = sc.sc().longAccumulator("TotalItems"); final LongAccumulator totalItems = sc.sc().longAccumulator("TotalItems");
final LongAccumulator invalidRecords = sc.sc().longAccumulator("InvalidRecords");
final LongAccumulator invalidRecords = sc.sc().longAccumulator("InvalidRecords"); final MessageManager manager = new MessageManager(
parser.get("rabbitHost"),
parser.get("rabbitUser"),
parser.get("rabbitPassword"),
false,
false,
null);
final MessageManager manager = new MessageManager( final JavaRDD<MetadataRecord> mappeRDD = inputRDD
parser.get("rabbitHost"), .map(
parser.get("rabbitUser"), item -> parseRecord(
parser.get("rabbitPassword"), item._2().toString(),
false, parser.get("xpath"),
false, parser.get("encoding"),
null); provenance,
dateOfCollection,
totalItems,
invalidRecords))
.filter(Objects::nonNull)
.distinct();
final JavaRDD<MetadataRecord> mappeRDD = inputRDD ongoingMap.put("ongoing", "0");
.map( if (!test) {
item -> parseRecord( manager
item._2().toString(), .sendMessage(
parser.get("xpath"), new Message(
parser.get("encoding"), parser.get("workflowId"), "DataFrameCreation", MessageType.ONGOING, ongoingMap),
provenance, parser.get("rabbitOngoingQueue"),
dateOfCollection, true,
totalItems, false);
invalidRecords)) }
.filter(Objects::nonNull)
.distinct();
ongoingMap.put("ongoing", "0"); final Encoder<MetadataRecord> encoder = Encoders.bean(MetadataRecord.class);
if (!test) { final Dataset<MetadataRecord> mdstore = spark.createDataset(mappeRDD.rdd(), encoder);
manager final LongAccumulator mdStoreRecords = sc.sc().longAccumulator("MDStoreRecords");
.sendMessage( mdStoreRecords.add(mdstore.count());
new Message( ongoingMap.put("ongoing", "" + totalItems.value());
parser.get("workflowId"), "DataFrameCreation", MessageType.ONGOING, ongoingMap), if (!test) {
parser.get("rabbitOngoingQueue"), manager
true, .sendMessage(
false); new Message(
} parser.get("workflowId"), "DataFrameCreation", MessageType.ONGOING, ongoingMap),
parser.get("rabbitOngoingQueue"),
true,
false);
}
mdstore.write().format("parquet").save(parser.get("output"));
reportMap.put("inputItem", "" + totalItems.value());
reportMap.put("invalidRecords", "" + invalidRecords.value());
reportMap.put("mdStoreSize", "" + mdStoreRecords.value());
if (!test) {
manager
.sendMessage(
new Message(parser.get("workflowId"), "Collection", MessageType.REPORT, reportMap),
parser.get("rabbitReportQueue"),
true,
false);
manager.close();
}
});
final Encoder<MetadataRecord> encoder = Encoders.bean(MetadataRecord.class);
final Dataset<MetadataRecord> mdstore = spark.createDataset(mappeRDD.rdd(), encoder);
final LongAccumulator mdStoreRecords = sc.sc().longAccumulator("MDStoreRecords");
mdStoreRecords.add(mdstore.count());
ongoingMap.put("ongoing", "" + totalItems.value());
if (!test) {
manager
.sendMessage(
new Message(
parser.get("workflowId"), "DataFrameCreation", MessageType.ONGOING, ongoingMap),
parser.get("rabbitOngoingQueue"),
true,
false);
}
mdstore.write().format("parquet").save(parser.get("output"));
reportMap.put("inputItem", "" + totalItems.value());
reportMap.put("invalidRecords", "" + invalidRecords.value());
reportMap.put("mdStoreSize", "" + mdStoreRecords.value());
if (!test) {
manager
.sendMessage(
new Message(parser.get("workflowId"), "Collection", MessageType.REPORT, reportMap),
parser.get("rabbitReportQueue"),
true,
false);
manager.close();
}
} }
} }

97
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java
View File

@ -1,13 +1,17 @@
package eu.dnetlib.dhp.transformation; package eu.dnetlib.dhp.transformation;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Optional;
import org.apache.commons.cli.*; import org.apache.commons.cli.*;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Encoder;
import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Encoders;
@ -17,8 +21,11 @@ import org.dom4j.Document;
import org.dom4j.DocumentException; import org.dom4j.DocumentException;
import org.dom4j.Node; import org.dom4j.Node;
import org.dom4j.io.SAXReader; import org.dom4j.io.SAXReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.collection.GenerateNativeStoreSparkJob;
import eu.dnetlib.dhp.model.mdstore.MetadataRecord; import eu.dnetlib.dhp.model.mdstore.MetadataRecord;
import eu.dnetlib.dhp.transformation.vocabulary.Vocabulary; import eu.dnetlib.dhp.transformation.vocabulary.Vocabulary;
import eu.dnetlib.dhp.transformation.vocabulary.VocabularyHelper; import eu.dnetlib.dhp.transformation.vocabulary.VocabularyHelper;
@ -29,6 +36,8 @@ import eu.dnetlib.message.MessageType;
public class TransformSparkJobNode { public class TransformSparkJobNode {
private static final Logger log = LoggerFactory.getLogger(TransformSparkJobNode.class);
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
final ArgumentApplicationParser parser = new ArgumentApplicationParser( final ArgumentApplicationParser parser = new ArgumentApplicationParser(
@ -40,12 +49,18 @@ public class TransformSparkJobNode {
parser.parseArgument(args); parser.parseArgument(args);
Boolean isSparkSessionManaged = Optional
.ofNullable(parser.get("isSparkSessionManaged"))
.map(Boolean::valueOf)
.orElse(Boolean.TRUE);
log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
final String inputPath = parser.get("input"); final String inputPath = parser.get("input");
final String outputPath = parser.get("output"); final String outputPath = parser.get("output");
final String workflowId = parser.get("workflowId"); final String workflowId = parser.get("workflowId");
final String trasformationRule = extractXSLTFromTR( final String trasformationRule = extractXSLTFromTR(
Objects.requireNonNull(DHPUtils.decompressString(parser.get("transformationRule")))); Objects.requireNonNull(DHPUtils.decompressString(parser.get("transformationRule"))));
final String master = parser.get("master");
final String rabbitUser = parser.get("rabbitUser"); final String rabbitUser = parser.get("rabbitUser");
final String rabbitPassword = parser.get("rabbitPassword"); final String rabbitPassword = parser.get("rabbitPassword");
final String rabbitHost = parser.get("rabbitHost"); final String rabbitHost = parser.get("rabbitHost");
@ -53,46 +68,48 @@ public class TransformSparkJobNode {
final long dateOfCollection = new Long(parser.get("dateOfCollection")); final long dateOfCollection = new Long(parser.get("dateOfCollection"));
final boolean test = parser.get("isTest") == null ? false : Boolean.valueOf(parser.get("isTest")); final boolean test = parser.get("isTest") == null ? false : Boolean.valueOf(parser.get("isTest"));
final SparkSession spark = SparkSession SparkConf conf = new SparkConf();
.builder() runWithSparkSession(
.appName("TransformStoreSparkJob") conf,
.master(master) isSparkSessionManaged,
.getOrCreate(); spark -> {
final Encoder<MetadataRecord> encoder = Encoders.bean(MetadataRecord.class);
final Dataset<MetadataRecord> mdstoreInput = spark.read().format("parquet").load(inputPath).as(encoder);
final LongAccumulator totalItems = spark.sparkContext().longAccumulator("TotalItems");
final LongAccumulator errorItems = spark.sparkContext().longAccumulator("errorItems");
final LongAccumulator transformedItems = spark.sparkContext().longAccumulator("transformedItems");
final Map<String, Vocabulary> vocabularies = new HashMap<>();
vocabularies.put("dnet:languages", VocabularyHelper.getVocabularyFromAPI("dnet:languages"));
final TransformFunction transformFunction = new TransformFunction(
totalItems,
errorItems,
transformedItems,
trasformationRule,
dateOfCollection,
vocabularies);
mdstoreInput.map(transformFunction, encoder).write().format("parquet").save(outputPath);
if (rabbitHost != null) {
System.out.println("SEND FINAL REPORT");
final Map<String, String> reportMap = new HashMap<>();
reportMap.put("inputItem", "" + totalItems.value());
reportMap.put("invalidRecords", "" + errorItems.value());
reportMap.put("mdStoreSize", "" + transformedItems.value());
System.out.println(new Message(workflowId, "Transform", MessageType.REPORT, reportMap));
if (!test) {
final MessageManager manager = new MessageManager(rabbitHost, rabbitUser, rabbitPassword, false,
false,
null);
manager
.sendMessage(
new Message(workflowId, "Transform", MessageType.REPORT, reportMap),
rabbitReportQueue,
true,
false);
manager.close();
}
}
});
final Encoder<MetadataRecord> encoder = Encoders.bean(MetadataRecord.class);
final Dataset<MetadataRecord> mdstoreInput = spark.read().format("parquet").load(inputPath).as(encoder);
final LongAccumulator totalItems = spark.sparkContext().longAccumulator("TotalItems");
final LongAccumulator errorItems = spark.sparkContext().longAccumulator("errorItems");
final LongAccumulator transformedItems = spark.sparkContext().longAccumulator("transformedItems");
final Map<String, Vocabulary> vocabularies = new HashMap<>();
vocabularies.put("dnet:languages", VocabularyHelper.getVocabularyFromAPI("dnet:languages"));
final TransformFunction transformFunction = new TransformFunction(
totalItems,
errorItems,
transformedItems,
trasformationRule,
dateOfCollection,
vocabularies);
mdstoreInput.map(transformFunction, encoder).write().format("parquet").save(outputPath);
if (rabbitHost != null) {
System.out.println("SEND FINAL REPORT");
final Map<String, String> reportMap = new HashMap<>();
reportMap.put("inputItem", "" + totalItems.value());
reportMap.put("invalidRecords", "" + errorItems.value());
reportMap.put("mdStoreSize", "" + transformedItems.value());
System.out.println(new Message(workflowId, "Transform", MessageType.REPORT, reportMap));
if (!test) {
final MessageManager manager = new MessageManager(rabbitHost, rabbitUser, rabbitPassword, false, false,
null);
manager
.sendMessage(
new Message(workflowId, "Transform", MessageType.REPORT, reportMap),
rabbitReportQueue,
true,
false);
manager.close();
}
}
} }
private static String extractXSLTFromTR(final String tr) throws DocumentException { private static String extractXSLTFromTR(final String tr) throws DocumentException {

98
dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/collection_input_parameters.json
View File

@ -1,16 +1,86 @@
[ [
{"paramName":"mt", "paramLongName":"master", "paramDescription": "should be local or yarn", "paramRequired": true}, {
{"paramName":"e", "paramLongName":"encoding", "paramDescription": "the encoding of the input record should be JSON or XML", "paramRequired": true}, "paramName": "issm",
{"paramName":"d", "paramLongName":"dateOfCollection", "paramDescription": "the date when the record has been stored", "paramRequired": true}, "paramLongName": "isSparkSessionManaged",
{"paramName":"p", "paramLongName":"provenance", "paramDescription": "the infos about the provenance of the collected records", "paramRequired": true}, "paramDescription": "when true will stop SparkSession after job execution",
{"paramName":"x", "paramLongName":"xpath", "paramDescription": "the xpath to identify the record ifentifier", "paramRequired": true}, "paramRequired": false
{"paramName":"i", "paramLongName":"input", "paramDescription": "the path of the sequencial file to read", "paramRequired": true}, },
{"paramName":"o", "paramLongName":"output", "paramDescription": "the path of the result DataFrame on HDFS", "paramRequired": true}, {
{"paramName":"ru", "paramLongName":"rabbitUser", "paramDescription": "the user to connect with RabbitMq for messaging", "paramRequired": true}, "paramName": "e",
{"paramName":"rp", "paramLongName":"rabbitPassword", "paramDescription": "the password to connect with RabbitMq for messaging", "paramRequired": true}, "paramLongName": "encoding",
{"paramName":"rh", "paramLongName":"rabbitHost", "paramDescription": "the host of the RabbitMq server", "paramRequired": true}, "paramDescription": "the encoding of the input record should be JSON or XML",
{"paramName":"ro", "paramLongName":"rabbitOngoingQueue", "paramDescription": "the name of the ongoing queue", "paramRequired": true}, "paramRequired": true
{"paramName":"rr", "paramLongName":"rabbitReportQueue", "paramDescription": "the name of the report queue", "paramRequired": true}, },
{"paramName":"w", "paramLongName":"workflowId", "paramDescription": "the identifier of the dnet Workflow", "paramRequired": true}, {
{"paramName":"t", "paramLongName":"isTest", "paramDescription": "the name of the report queue", "paramRequired": false} "paramName": "d",
"paramLongName": "dateOfCollection",
"paramDescription": "the date when the record has been stored",
"paramRequired": true
},
{
"paramName": "p",
"paramLongName": "provenance",
"paramDescription": "the infos about the provenance of the collected records",
"paramRequired": true
},
{
"paramName": "x",
"paramLongName": "xpath",
"paramDescription": "the xpath to identify the record identifier",
"paramRequired": true
},
{
"paramName": "i",
"paramLongName": "input",
"paramDescription": "the path of the sequencial file to read",
"paramRequired": true
},
{
"paramName": "o",
"paramLongName": "output",
"paramDescription": "the path of the result DataFrame on HDFS",
"paramRequired": true
},
{
"paramName": "ru",
"paramLongName": "rabbitUser",
"paramDescription": "the user to connect with RabbitMq for messaging",
"paramRequired": true
},
{
"paramName": "rp",
"paramLongName": "rabbitPassword",
"paramDescription": "the password to connect with RabbitMq for messaging",
"paramRequired": true
},
{
"paramName": "rh",
"paramLongName": "rabbitHost",
"paramDescription": "the host of the RabbitMq server",
"paramRequired": true
},
{
"paramName": "ro",
"paramLongName": "rabbitOngoingQueue",
"paramDescription": "the name of the ongoing queue",
"paramRequired": true
},
{
"paramName": "rr",
"paramLongName": "rabbitReportQueue",
"paramDescription": "the name of the report queue",
"paramRequired": true
},
{
"paramName": "w",
"paramLongName": "workflowId",
"paramDescription": "the identifier of the dnet Workflow",
"paramRequired": true
},
{
"paramName": "t",
"paramLongName": "isTest",
"paramDescription": "the name of the report queue",
"paramRequired": false
}
] ]

86
dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/transformation/transformation_input_parameters.json
View File

@ -1,16 +1,74 @@
[ [
{"paramName":"mt", "paramLongName":"master", "paramDescription": "should be local or yarn", "paramRequired": true}, {
{"paramName":"d", "paramLongName":"dateOfCollection", "paramDescription": "the date when the record has been stored", "paramRequired": true}, "paramName": "issm",
{"paramName":"i", "paramLongName":"input", "paramDescription": "the path of the sequencial file to read", "paramRequired": true}, "paramLongName": "isSparkSessionManaged",
{"paramName":"o", "paramLongName":"output", "paramDescription": "the path of the result DataFrame on HDFS", "paramRequired": true}, "paramDescription": "when true will stop SparkSession after job execution",
{"paramName":"w", "paramLongName":"workflowId", "paramDescription": "the identifier of the dnet Workflow", "paramRequired": true}, "paramRequired": false
{"paramName":"tr", "paramLongName":"transformationRule","paramDescription": "the transformation Rule to apply to the input MDStore", "paramRequired": true}, },
{"paramName":"ru", "paramLongName":"rabbitUser", "paramDescription": "the user to connect with RabbitMq for messaging", "paramRequired": true}, {
{"paramName":"rp", "paramLongName":"rabbitPassword", "paramDescription": "the password to connect with RabbitMq for messaging", "paramRequired": true}, "paramName": "d",
{"paramName":"rh", "paramLongName":"rabbitHost", "paramDescription": "the host of the RabbitMq server", "paramRequired": true}, "paramLongName": "dateOfCollection",
{"paramName":"ro", "paramLongName":"rabbitOngoingQueue", "paramDescription": "the name of the ongoing queue", "paramRequired": true}, "paramDescription": "the date when the record has been stored",
{"paramName":"rr", "paramLongName":"rabbitReportQueue", "paramDescription": "the name of the report queue", "paramRequired": true}, "paramRequired": true
{"paramName":"t", "paramLongName":"isTest", "paramDescription": "the name of the report queue", "paramRequired": false} },
{
"paramName": "i",
"paramLongName": "input",
"paramDescription": "the path of the sequencial file to read",
"paramRequired": true
},
{
"paramName": "o",
"paramLongName": "output",
"paramDescription": "the path of the result DataFrame on HDFS",
"paramRequired": true
},
{
"paramName": "w",
"paramLongName": "workflowId",
"paramDescription": "the identifier of the dnet Workflow",
"paramRequired": true
},
{
"paramName": "tr",
"paramLongName": "transformationRule",
"paramDescription": "the transformation Rule to apply to the input MDStore",
"paramRequired": true
},
{
"paramName": "ru",
"paramLongName": "rabbitUser",
"paramDescription": "the user to connect with RabbitMq for messaging",
"paramRequired": true
},
{
"paramName": "rp",
"paramLongName": "rabbitPassword",
"paramDescription": "the password to connect with RabbitMq for messaging",
"paramRequired": true
},
{
"paramName": "rh",
"paramLongName": "rabbitHost",
"paramDescription": "the host of the RabbitMq server",
"paramRequired": true
},
{
"paramName": "ro",
"paramLongName": "rabbitOngoingQueue",
"paramDescription": "the name of the ongoing queue",
"paramRequired": true
},
{
"paramName": "rr",
"paramLongName": "rabbitReportQueue",
"paramDescription": "the name of the report queue",
"paramRequired": true
},
{
"paramName": "t",
"paramLongName": "isTest",
"paramDescription": "the name of the report queue",
"paramRequired": false
}
] ]

86
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/CollectionJobTest.java
View File

@ -9,65 +9,60 @@ import java.nio.file.Path;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.AfterEach; import org.apache.spark.SparkConf;
import org.junit.jupiter.api.BeforeEach; import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.*;
import org.junit.jupiter.api.io.TempDir;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.model.mdstore.MetadataRecord; import eu.dnetlib.dhp.model.mdstore.MetadataRecord;
import eu.dnetlib.dhp.model.mdstore.Provenance; import eu.dnetlib.dhp.model.mdstore.Provenance;
import eu.dnetlib.dhp.schema.common.ModelSupport;
public class CollectionJobTest { public class CollectionJobTest {
private Path testDir; private static SparkSession spark;
@BeforeEach @BeforeAll
public void setup() throws IOException { public static void beforeAll() {
testDir = Files.createTempDirectory("dhp-collection"); SparkConf conf = new SparkConf();
conf.setAppName(CollectionJobTest.class.getSimpleName());
conf.setMaster("local");
spark = SparkSession.builder().config(conf).getOrCreate();
} }
@AfterEach @AfterAll
public void teadDown() throws IOException { public static void afterAll() {
FileUtils.deleteDirectory(testDir.toFile()); spark.stop();
} }
@Test @Test
public void tesCollection() throws Exception { public void tesCollection(@TempDir Path testDir) throws Exception {
final Provenance provenance = new Provenance("pippo", "puppa", "ns_prefix"); final Provenance provenance = new Provenance("pippo", "puppa", "ns_prefix");
Assertions.assertNotNull(new ObjectMapper().writeValueAsString(provenance));
GenerateNativeStoreSparkJob GenerateNativeStoreSparkJob
.main( .main(
new String[] { new String[] {
"-mt", "issm", "true",
"local", "-w", "wid",
"-w", "-e", "XML",
"wid", "-d", "" + System.currentTimeMillis(),
"-e", "-p", new ObjectMapper().writeValueAsString(provenance),
"XML", "-x", "./*[local-name()='record']/*[local-name()='header']/*[local-name()='identifier']",
"-d", "-i", this.getClass().getResource("/eu/dnetlib/dhp/collection/native.seq").toString(),
"" + System.currentTimeMillis(), "-o", testDir.toString() + "/store",
"-p", "-t", "true",
new ObjectMapper().writeValueAsString(provenance), "-ru", "",
"-x", "-rp", "",
"./*[local-name()='record']/*[local-name()='header']/*[local-name()='identifier']", "-rh", "",
"-i", "-ro", "",
this.getClass().getResource("/eu/dnetlib/dhp/collection/native.seq").toString(), "-rr", ""
"-o",
testDir.toString() + "/store",
"-t",
"true",
"-ru",
"",
"-rp",
"",
"-rh",
"",
"-ro",
"",
"-rr",
""
}); });
System.out.println(new ObjectMapper().writeValueAsString(provenance));
// TODO introduce useful assertions
} }
@Test @Test
@ -85,9 +80,8 @@ public class CollectionJobTest {
null, null,
null); null);
assert record != null; assertNotNull(record.getId());
System.out.println(record.getId()); assertNotNull(record.getOriginalId());
System.out.println(record.getOriginalId());
} }
@Test @Test
@ -112,10 +106,12 @@ public class CollectionJobTest {
System.currentTimeMillis(), System.currentTimeMillis(),
null, null,
null); null);
assert record != null;
record.setBody("ciao"); record.setBody("ciao");
assert record1 != null;
record1.setBody("mondo"); record1.setBody("mondo");
assertNotNull(record);
assertNotNull(record1);
assertEquals(record, record1); assertEquals(record, record1);
} }
} }

58
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/transformation/TransformationJobTest.java
View File

@ -12,10 +12,14 @@ import java.util.Map;
import javax.xml.transform.stream.StreamSource; import javax.xml.transform.stream.StreamSource;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.util.LongAccumulator; import org.apache.spark.util.LongAccumulator;
import org.dom4j.Document; import org.dom4j.Document;
import org.dom4j.Node; import org.dom4j.Node;
import org.dom4j.io.SAXReader; import org.dom4j.io.SAXReader;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
@ -23,6 +27,7 @@ import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import eu.dnetlib.dhp.collection.CollectionJobTest;
import eu.dnetlib.dhp.model.mdstore.MetadataRecord; import eu.dnetlib.dhp.model.mdstore.MetadataRecord;
import eu.dnetlib.dhp.transformation.functions.Cleaner; import eu.dnetlib.dhp.transformation.functions.Cleaner;
import eu.dnetlib.dhp.transformation.vocabulary.Vocabulary; import eu.dnetlib.dhp.transformation.vocabulary.Vocabulary;
@ -33,6 +38,21 @@ import net.sf.saxon.s9api.*;
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
public class TransformationJobTest { public class TransformationJobTest {
private static SparkSession spark;
@BeforeAll
public static void beforeAll() {
SparkConf conf = new SparkConf();
conf.setAppName(CollectionJobTest.class.getSimpleName());
conf.setMaster("local");
spark = SparkSession.builder().config(conf).getOrCreate();
}
@AfterAll
public static void afterAll() {
spark.stop();
}
@Mock @Mock
private LongAccumulator accumulator; private LongAccumulator accumulator;
@ -78,31 +98,21 @@ public class TransformationJobTest {
TransformSparkJobNode TransformSparkJobNode
.main( .main(
new String[] { new String[] {
"-mt", "-issm", "true",
"local", "-i", mdstore_input,
"-i", "-o", mdstore_output,
mdstore_input, "-d", "1",
"-o", "-w", "1",
mdstore_output, "-tr", xslt,
"-d", "-t", "true",
"1", "-ru", "",
"-w", "-rp", "",
"1", "-rh", "",
"-tr", "-ro", "",
xslt, "-rr", ""
"-t",
"true",
"-ru",
"",
"-rp",
"",
"-rh",
"",
"-ro",
"",
"-rr",
""
}); });
// TODO introduce useful assertions
} }
@Test @Test

2
dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java
View File

@ -94,7 +94,7 @@ public class SparkPropagateRelation extends AbstractSparkAction {
FieldType.TARGET, FieldType.TARGET,
getDeletedFn()); getDeletedFn());
save(newRels.union(updated), outputRelationPath, SaveMode.Overwrite); save(newRels.union(updated).union(mergeRels), outputRelationPath, SaveMode.Overwrite);
} }
private static Dataset<Relation> processDataset( private static Dataset<Relation> processDataset(

2
dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
View File

@ -420,7 +420,7 @@ public class SparkDedupTest implements Serializable {
long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count(); long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count();
assertEquals(826, relations); assertEquals(5022, relations);
// check deletedbyinference // check deletedbyinference
final Dataset<Relation> mergeRels = spark final Dataset<Relation> mergeRels = spark

70
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml
View File

@ -115,11 +115,11 @@
<delete path="${contentPath}/db_claims"/> <delete path="${contentPath}/db_claims"/>
</prepare> </prepare>
<main-class>eu.dnetlib.dhp.oa.graph.raw.MigrateDbEntitiesApplication</main-class> <main-class>eu.dnetlib.dhp.oa.graph.raw.MigrateDbEntitiesApplication</main-class>
<arg>-p</arg><arg>${contentPath}/db_claims</arg> <arg>--hdfsPath</arg><arg>${contentPath}/db_claims</arg>
<arg>-pgurl</arg><arg>${postgresURL}</arg> <arg>--postgresUrl</arg><arg>${postgresURL}</arg>
<arg>-pguser</arg><arg>${postgresUser}</arg> <arg>--postgresUser</arg><arg>${postgresUser}</arg>
<arg>-pgpasswd</arg><arg>${postgresPassword}</arg> <arg>--postgresPassword</arg><arg>${postgresPassword}</arg>
<arg>-a</arg><arg>claims</arg> <arg>--action</arg><arg>claims</arg>
</java> </java>
<ok to="ImportODF_claims"/> <ok to="ImportODF_claims"/>
<error to="Kill"/> <error to="Kill"/>
@ -165,10 +165,10 @@
<delete path="${contentPath}/db_records"/> <delete path="${contentPath}/db_records"/>
</prepare> </prepare>
<main-class>eu.dnetlib.dhp.oa.graph.raw.MigrateDbEntitiesApplication</main-class> <main-class>eu.dnetlib.dhp.oa.graph.raw.MigrateDbEntitiesApplication</main-class>
<arg>-p</arg><arg>${contentPath}/db_records</arg> <arg>--hdfsPath</arg><arg>${contentPath}/db_records</arg>
<arg>-pgurl</arg><arg>${postgresURL}</arg> <arg>--postgresUrl</arg><arg>${postgresURL}</arg>
<arg>-pguser</arg><arg>${postgresUser}</arg> <arg>--postgresUser</arg><arg>${postgresUser}</arg>
<arg>-pgpasswd</arg><arg>${postgresPassword}</arg> <arg>--postgresPassword</arg><arg>${postgresPassword}</arg>
</java> </java>
<ok to="ImportODF"/> <ok to="ImportODF"/>
<error to="Kill"/> <error to="Kill"/>
@ -180,12 +180,12 @@
<delete path="${contentPath}/odf_records"/> <delete path="${contentPath}/odf_records"/>
</prepare> </prepare>
<main-class>eu.dnetlib.dhp.oa.graph.raw.MigrateMongoMdstoresApplication</main-class> <main-class>eu.dnetlib.dhp.oa.graph.raw.MigrateMongoMdstoresApplication</main-class>
<arg>-p</arg><arg>${contentPath}/odf_records</arg> <arg>--hdfsPath</arg><arg>${contentPath}/odf_records</arg>
<arg>-mongourl</arg><arg>${mongoURL}</arg> <arg>--mongoBaseUrl</arg><arg>${mongoURL}</arg>
<arg>-mongodb</arg><arg>${mongoDb}</arg> <arg>--mongoDb</arg><arg>${mongoDb}</arg>
<arg>-f</arg><arg>ODF</arg> <arg>--mdFormat</arg><arg>ODF</arg>
<arg>-l</arg><arg>store</arg> <arg>--mdLayout</arg><arg>store</arg>
<arg>-i</arg><arg>cleaned</arg> <arg>--mdInterpretation</arg><arg>cleaned</arg>
</java> </java>
<ok to="ImportOAF"/> <ok to="ImportOAF"/>
<error to="Kill"/> <error to="Kill"/>
@ -197,12 +197,12 @@
<delete path="${contentPath}/oaf_records"/> <delete path="${contentPath}/oaf_records"/>
</prepare> </prepare>
<main-class>eu.dnetlib.dhp.oa.graph.raw.MigrateMongoMdstoresApplication</main-class> <main-class>eu.dnetlib.dhp.oa.graph.raw.MigrateMongoMdstoresApplication</main-class>
<arg>-p</arg><arg>${contentPath}/oaf_records</arg> <arg>--hdfsPath</arg><arg>${contentPath}/oaf_records</arg>
<arg>-mongourl</arg><arg>${mongoURL}</arg> <arg>--mongoBaseUrl</arg><arg>${mongoURL}</arg>
<arg>-mongodb</arg><arg>${mongoDb}</arg> <arg>--mongoDb</arg><arg>${mongoDb}</arg>
<arg>-f</arg><arg>OAF</arg> <arg>--mdFormat</arg><arg>OAF</arg>
<arg>-l</arg><arg>store</arg> <arg>--mdLayout</arg><arg>store</arg>
<arg>-i</arg><arg>cleaned</arg> <arg>--mdInterpretation</arg><arg>cleaned</arg>
</java> </java>
<ok to="wait_import"/> <ok to="wait_import"/>
<error to="Kill"/> <error to="Kill"/>
@ -231,11 +231,11 @@
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
</spark-opts> </spark-opts>
<arg>-s</arg><arg>${contentPath}/db_claims,${contentPath}/oaf_claims,${contentPath}/odf_claims</arg> <arg>--sourcePaths</arg><arg>${contentPath}/db_claims,${contentPath}/oaf_claims,${contentPath}/odf_claims</arg>
<arg>-t</arg><arg>${workingDir}/entities_claim</arg> <arg>--targetPath</arg><arg>${workingDir}/entities_claim</arg>
<arg>-pgurl</arg><arg>${postgresURL}</arg> <arg>--postgresUrl</arg><arg>${postgresURL}</arg>
<arg>-pguser</arg><arg>${postgresUser}</arg> <arg>--postgresUser</arg><arg>${postgresUser}</arg>
<arg>-pgpasswd</arg><arg>${postgresPassword}</arg> <arg>--postgresPassword</arg><arg>${postgresPassword}</arg>
</spark> </spark>
<ok to="GenerateGraph_claims"/> <ok to="GenerateGraph_claims"/>
<error to="Kill"/> <error to="Kill"/>
@ -257,8 +257,8 @@
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
</spark-opts> </spark-opts>
<arg>-s</arg><arg>${workingDir}/entities_claim</arg> <arg>--sourcePath</arg><arg>${workingDir}/entities_claim</arg>
<arg>-g</arg><arg>${workingDir}/graph_claims</arg> <arg>--graphRawPath</arg><arg>${workingDir}/graph_claims</arg>
</spark> </spark>
<ok to="wait_graphs"/> <ok to="wait_graphs"/>
<error to="Kill"/> <error to="Kill"/>
@ -280,11 +280,11 @@
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
</spark-opts> </spark-opts>
<arg>-s</arg><arg>${contentPath}/db_records,${contentPath}/oaf_records,${contentPath}/odf_records</arg> <arg>--sourcePaths</arg><arg>${contentPath}/db_records,${contentPath}/oaf_records,${contentPath}/odf_records</arg>
<arg>-t</arg><arg>${workingDir}/entities</arg> <arg>--targetPath</arg><arg>${workingDir}/entities</arg>
<arg>-pgurl</arg><arg>${postgresURL}</arg> <arg>--postgresUrl</arg><arg>${postgresURL}</arg>
<arg>-pguser</arg><arg>${postgresUser}</arg> <arg>--postgresUser</arg><arg>${postgresUser}</arg>
<arg>-pgpasswd</arg><arg>${postgresPassword}</arg> <arg>--postgresPassword</arg><arg>${postgresPassword}</arg>
</spark> </spark>
<ok to="GenerateGraph"/> <ok to="GenerateGraph"/>
<error to="Kill"/> <error to="Kill"/>
@ -307,8 +307,8 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.shuffle.partitions=7680 --conf spark.sql.shuffle.partitions=7680
</spark-opts> </spark-opts>
<arg>-s</arg><arg>${workingDir}/entities</arg> <arg>--sourcePath</arg><arg>${workingDir}/entities</arg>
<arg>-g</arg><arg>${workingDir}/graph_raw</arg> <arg>--graphRawPath</arg><arg>${workingDir}/graph_raw</arg>
</spark> </spark>
<ok to="wait_graphs"/> <ok to="wait_graphs"/>
<error to="Kill"/> <error to="Kill"/>

85
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_db/oozie_app/workflow.xml
View File

@ -1,8 +1,8 @@
<workflow-app name="import db entities (step 1)" xmlns="uri:oozie:workflow:0.5"> <workflow-app name="import DB entities" xmlns="uri:oozie:workflow:0.5">
<parameters> <parameters>
<property> <property>
<name>migrationPathStep1</name> <name>contentPath</name>
<description>the base path to store hdfs file</description> <description>path location to store (or reuse) content from the aggregator</description>
</property> </property>
<property> <property>
<name>postgresURL</name> <name>postgresURL</name>
@ -16,6 +16,7 @@
<name>postgresPassword</name> <name>postgresPassword</name>
<description>the password postgres</description> <description>the password postgres</description>
</property> </property>
<property> <property>
<name>sparkDriverMemory</name> <name>sparkDriverMemory</name>
<description>memory for driver process</description> <description>memory for driver process</description>
@ -28,31 +29,81 @@
<name>sparkExecutorCores</name> <name>sparkExecutorCores</name>
<description>number of cores used by single executor</description> <description>number of cores used by single executor</description>
</property> </property>
<property>
<name>oozieActionShareLibForSpark2</name>
<description>oozie action sharelib for spark 2.*</description>
</property>
<property>
<name>spark2ExtraListeners</name>
<value>com.cloudera.spark.lineage.NavigatorAppListener</value>
<description>spark 2.* extra listeners classname</description>
</property>
<property>
<name>spark2SqlQueryExecutionListeners</name>
<value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
<description>spark 2.* sql query execution listeners classname</description>
</property>
<property>
<name>spark2YarnHistoryServerAddress</name>
<description>spark 2.* yarn history server address</description>
</property>
<property>
<name>spark2EventLogDir</name>
<description>spark 2.* event log dir location</description>
</property>
</parameters> </parameters>
<start to="ResetWorkingPath"/> <global>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapreduce.job.queuename</name>
<value>${queueName}</value>
</property>
<property>
<name>oozie.launcher.mapred.job.queue.name</name>
<value>${oozieLauncherQueueName}</value>
</property>
<property>
<name>oozie.action.sharelib.for.spark</name>
<value>${oozieActionShareLibForSpark2}</value>
</property>
</configuration>
</global>
<start to="ImportDB"/>
<kill name="Kill"> <kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
</kill> </kill>
<action name="ResetWorkingPath"> <action name="ImportDB">
<fs> <java>
<delete path='${migrationPathStep1}/db_records'/> <prepare>
</fs> <delete path="${contentPath}/db_records"/>
<ok to="ImportDB"/> </prepare>
<main-class>eu.dnetlib.dhp.oa.graph.raw.MigrateDbEntitiesApplication</main-class>
<arg>--hdfsPath</arg><arg>${contentPath}/db_records</arg>
<arg>--postgresUrl</arg><arg>${postgresURL}</arg>
<arg>--postgresUser</arg><arg>${postgresUser}</arg>
<arg>--postgresPassword</arg><arg>${postgresPassword}</arg>
</java>
<ok to="ImportDB_claims"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>
<action name="ImportDB"> <action name="ImportDB_claims">
<java> <java>
<job-tracker>${jobTracker}</job-tracker> <prepare>
<name-node>${nameNode}</name-node> <delete path="${contentPath}/db_claims"/>
<main-class>eu.dnetlib.dhp.migration.step1.MigrateDbEntitiesApplication</main-class> </prepare>
<arg>-p</arg><arg>${migrationPathStep1}/db_records</arg> <main-class>eu.dnetlib.dhp.oa.graph.raw.MigrateDbEntitiesApplication</main-class>
<arg>-pgurl</arg><arg>${postgresURL}</arg> <arg>--hdfsPath</arg><arg>${contentPath}/db_claims</arg>
<arg>-pguser</arg><arg>${postgresUser}</arg> <arg>--postgresUrl</arg><arg>${postgresURL}</arg>
<arg>-pgpasswd</arg><arg>${postgresPassword}</arg> <arg>--postgresUser</arg><arg>${postgresUser}</arg>
<arg>--postgresPassword</arg><arg>${postgresPassword}</arg>
<arg>--action</arg><arg>claims</arg>
</java> </java>
<ok to="End"/> <ok to="End"/>
<error to="Kill"/> <error to="Kill"/>

4
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJob.java
View File

@ -113,7 +113,7 @@ public class XmlIndexingJob {
}); });
} }
private static String toIndexRecord(Transformer tr, final String record) { protected static String toIndexRecord(Transformer tr, final String record) {
final StreamResult res = new StreamResult(new StringWriter()); final StreamResult res = new StreamResult(new StringWriter());
try { try {
tr.transform(new StreamSource(new StringReader(record)), res); tr.transform(new StreamSource(new StringReader(record)), res);
@ -135,7 +135,7 @@ public class XmlIndexingJob {
* @throws IOException could happen * @throws IOException could happen
* @throws TransformerException could happen * @throws TransformerException could happen
*/ */
private static String getLayoutTransformer(String format, String fields, String xslt) protected static String getLayoutTransformer(String format, String fields, String xslt)
throws TransformerException { throws TransformerException {
final Transformer layoutTransformer = SaxonTransformerFactory.newInstance(xslt); final Transformer layoutTransformer = SaxonTransformerFactory.newInstance(xslt);

60
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java
View File

@ -9,10 +9,7 @@ import java.io.IOException;
import java.io.Serializable; import java.io.Serializable;
import java.io.StringReader; import java.io.StringReader;
import java.io.StringWriter; import java.io.StringWriter;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import javax.xml.transform.*; import javax.xml.transform.*;
@ -47,7 +44,7 @@ import eu.dnetlib.dhp.schema.oaf.Result;
public class XmlRecordFactory implements Serializable { public class XmlRecordFactory implements Serializable {
public static final String REL_SUBTYPE_DEDUP = "dedup"; private static final String REL_SUBTYPE_DEDUP = "dedup";
private final Map<String, LongAccumulator> accumulators; private final Map<String, LongAccumulator> accumulators;
private final Set<String> specialDatasourceTypes; private final Set<String> specialDatasourceTypes;
@ -100,8 +97,8 @@ public class XmlRecordFactory implements Serializable {
final List<String> relations = je final List<String> relations = je
.getLinks() .getLinks()
.stream() .stream()
.filter(t -> !REL_SUBTYPE_DEDUP.equalsIgnoreCase(t.getRelation().getSubRelType())) .filter(link -> !isDuplicate(link))
.map(link -> mapRelation(link, templateFactory, contexts)) .map(link -> mapRelation(contexts, templateFactory, type, link))
.collect(Collectors.toCollection(ArrayList::new)); .collect(Collectors.toCollection(ArrayList::new));
final String mainType = ModelSupport.getMainType(type); final String mainType = ModelSupport.getMainType(type);
@ -936,7 +933,7 @@ public class XmlRecordFactory implements Serializable {
metadata.add(XmlSerializationUtils.mapQualifier("datasourcetypeui", dsType)); metadata.add(XmlSerializationUtils.mapQualifier("datasourcetypeui", dsType));
} }
private String mapRelation(Tuple2 link, TemplateFactory templateFactory, Set<String> contexts) { private List<String> mapFields(Tuple2 link, Set<String> contexts) {
final Relation rel = link.getRelation(); final Relation rel = link.getRelation();
final RelatedEntity re = link.getRelatedEntity(); final RelatedEntity re = link.getRelatedEntity();
final String targetType = link.getRelatedEntity().getType(); final String targetType = link.getRelatedEntity().getType();
@ -1040,38 +1037,47 @@ public class XmlRecordFactory implements Serializable {
default: default:
throw new IllegalArgumentException("invalid target type: " + targetType); throw new IllegalArgumentException("invalid target type: " + targetType);
} }
final DataInfo info = rel.getDataInfo();
final String scheme = ModelSupport.getScheme(re.getType(), targetType);
if (StringUtils.isBlank(scheme)) {
throw new IllegalArgumentException(
String.format("missing scheme for: <%s - %s>", re.getType(), targetType));
}
final String accumulatorName = getRelDescriptor(rel.getRelType(), rel.getSubRelType(), rel.getRelClass()); final String accumulatorName = getRelDescriptor(rel.getRelType(), rel.getSubRelType(), rel.getRelClass());
if (accumulators.containsKey(accumulatorName)) { if (accumulators.containsKey(accumulatorName)) {
accumulators.get(accumulatorName).add(1); accumulators.get(accumulatorName).add(1);
} }
return metadata;
}
private String mapRelation(Set<String> contexts, TemplateFactory templateFactory, EntityType type, Tuple2 link) {
final Relation rel = link.getRelation();
final String targetType = link.getRelatedEntity().getType();
final String scheme = ModelSupport.getScheme(type.toString(), targetType);
if (StringUtils.isBlank(scheme)) {
throw new IllegalArgumentException(
String.format("missing scheme for: <%s - %s>", type.toString(), targetType));
}
final HashSet<String> fields = Sets.newHashSet(mapFields(link, contexts));
return templateFactory return templateFactory
.getRel( .getRel(
targetType, rel.getTarget(), Sets.newHashSet(metadata), rel.getRelClass(), scheme, info); targetType, rel.getTarget(), fields, rel.getRelClass(), scheme, rel.getDataInfo());
} }
private List<String> listChildren( private List<String> listChildren(
final OafEntity entity, JoinedEntity je, TemplateFactory templateFactory) { final OafEntity entity, JoinedEntity je, TemplateFactory templateFactory) {
final List<String> children = Lists.newArrayList();
EntityType entityType = EntityType.valueOf(je.getEntity().getType()); EntityType entityType = EntityType.valueOf(je.getEntity().getType());
children List<String> children = je
.addAll( .getLinks()
je .stream()
.getLinks() .filter(link -> isDuplicate(link))
.stream() .map(link -> {
.filter(link -> REL_SUBTYPE_DEDUP.equalsIgnoreCase(link.getRelation().getSubRelType())) final String targetType = link.getRelatedEntity().getType();
.map(link -> mapRelation(link, templateFactory, null)) final String name = ModelSupport.getMainType(EntityType.valueOf(targetType));
.collect(Collectors.toCollection(ArrayList::new))); final HashSet<String> fields = Sets.newHashSet(mapFields(link, null));
return templateFactory
.getChild(name, link.getRelatedEntity().getId(), Lists.newArrayList(fields));
})
.collect(Collectors.toCollection(ArrayList::new));
if (MainEntityType.result.toString().equals(ModelSupport.getMainType(entityType))) { if (MainEntityType.result.toString().equals(ModelSupport.getMainType(entityType))) {
final List<Instance> instances = ((Result) entity).getInstance(); final List<Instance> instances = ((Result) entity).getInstance();
@ -1178,6 +1184,10 @@ public class XmlRecordFactory implements Serializable {
return children; return children;
} }
private boolean isDuplicate(Tuple2 link) {
return REL_SUBTYPE_DEDUP.equalsIgnoreCase(link.getRelation().getSubRelType());
}
private List<String> listExtraInfo(OafEntity entity) { private List<String> listExtraInfo(OafEntity entity) {
final List<ExtraInfo> extraInfo = entity.getExtraInfo(); final List<ExtraInfo> extraInfo = entity.getExtraInfo();
return extraInfo != null return extraInfo != null

6
dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/template/child.st
View File

@ -1,3 +1,3 @@
<name$if(hasId)$ objidentifier="$id$"$else$$endif>> <$name$$if(hasId)$ objidentifier="$id$"$else$$endif$>
$metadata:{ it | $it$ }$ $metadata:{$it$}$
</name> </$name$>

42
dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/GraphJoinerTest.java
View File

@ -1,42 +0,0 @@
package eu.dnetlib.dhp.oa.provision;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.jupiter.api.BeforeEach;
public class GraphJoinerTest {
private final ClassLoader cl = getClass().getClassLoader();
private Path workingDir;
private Path inputDir;
private Path outputDir;
@BeforeEach
public void before() throws IOException {
workingDir = Files.createTempDirectory("promote_action_set");
inputDir = workingDir.resolve("input");
outputDir = workingDir.resolve("output");
}
private static void copyFiles(Path source, Path target) throws IOException {
Files
.list(source)
.forEach(
f -> {
try {
if (Files.isDirectory(f)) {
Path subTarget = Files.createDirectories(target.resolve(f.getFileName()));
copyFiles(f, subTarget);
} else {
Files.copy(f, target.resolve(f.getFileName()));
}
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
});
}
}

39
dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java Normal file
View File

@ -0,0 +1,39 @@
package eu.dnetlib.dhp.oa.provision;
import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import eu.dnetlib.dhp.utils.saxon.SaxonTransformerFactory;
public class IndexRecordTransformerTest {
@Test
public void testTrasformRecord() throws IOException, TransformerException {
String fields = IOUtils.toString(getClass().getResourceAsStream("fields.xml"));
String record = IOUtils.toString(getClass().getResourceAsStream("record.xml"));
String xslt = IOUtils.toString(getClass().getResourceAsStream("layoutToRecordTransformer.xsl"));
String transformer = XmlIndexingJob.getLayoutTransformer("DMF", fields, xslt);
Transformer tr = SaxonTransformerFactory.newInstance(transformer);
String a = XmlIndexingJob.toIndexRecord(tr, record);
System.out.println(a);
}
}

166
dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml Normal file
View File

@ -0,0 +1,166 @@
<LAYOUT name="index">
<FIELDS>
<FIELD indexable="false" name="oafentity" result="true" stat="false" tokenizable="false" xpath="//*[local-name() = 'entity']"/>
<FIELD indexable="true" name="oaftype" result="false" stat="false" tokenizable="false" value="local-name(//*[local-name()='entity']/*[local-name() != 'extraInfo'])"/>
<FIELD indexable="true" name="objIdentifier" result="false" stat="false" tokenizable="false" xpath="//header/dri:objIdentifier"/><!-- DATASOURCE FIELDS -->
<FIELD indexable="true" name="datasourceofficialname" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/officialname"/>
<FIELD indexable="true" name="datasourceenglishname" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/englishname"/>
<FIELD indexable="true" name="datasourceoddescription" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/oddescription"/>
<FIELD indexable="true" name="datasourceodsubjects" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/odsubjects"/>
<FIELD indexable="true" name="datasourceodlanguages" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/odlanguages"/>
<FIELD indexable="true" name="datasourceodcontenttypes" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/odcontenttypes"/>
<FIELD indexable="true" multivalued="false" name="datasourcetypename" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/datasourcetype/@classname"/>
<FIELD indexable="true" multivalued="false" name="datasourcetypeuiid" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/datasourcetypeui/@classid"/>
<FIELD indexable="true" multivalued="false" name="datasourcetypeuiname" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/datasourcetypeui/@classname"/>
<FIELD indexable="true" multivalued="false" name="datasourcecompatibilityid" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/openairecompatibility/@classid"/>
<FIELD indexable="true" multivalued="false" name="datasourcecompatibilityname" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/openairecompatibility/@classname"/>
<FIELD indexable="true" multivalued="true" name="datasourcesubject" result="false" stat="false" type="ngramtext" xpath="//*[local-name()='entity']/*[local-name()='datasource']/subjects"/>
<FIELD indexable="true" name="versioning" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='datasource']/versioning"/><!-- ORGANIZATION FIELDS -->
<FIELD indexable="true" name="organizationlegalshortname" result="false" stat="false" type="ngramtext" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='organization']//legalshortname)"/>
<FIELD indexable="true" name="organizationlegalname" result="false" stat="false" type="ngramtext" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='organization']//legalname)"/>
<FIELD indexable="true" name="organizationalternativenames" result="false" stat="false" type="ngramtext" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='organization']//alternativeNames)"/>
<FIELD indexable="true" name="organizationeclegalbody" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='organization']/eclegalbody"/>
<FIELD indexable="true" name="organizationeclegalperson" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='organization']/eclegalperson"/>
<FIELD indexable="true" name="organizationecnonprofit" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='organization']/ecnonprofit"/>
<FIELD indexable="true" name="organizationecresearchorganization" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='organization']/ecresearchorganization"/>
<FIELD indexable="true" name="organizationecinternationalorganizationeurinterests" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='organization']/ecinternationalorganizationeurinterests"/>
<FIELD indexable="true" name="organizationecinternationalorganization" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='organization']/ecinternationalorganization"/>
<FIELD indexable="true" name="organizationecenterprise" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='organization']/ecenterprise"/>
<FIELD indexable="true" name="organizationecsmevalidated" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='organization']/ecsmevalidated"/>
<FIELD indexable="true" name="organizationecnutscode" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='organization']/ecnutscode"/>
<FIELD indexable="true" multivalued="false" name="organizationcountryname" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='organization']/country/@classname"/><!-- PROJECT FIELDS -->
<FIELD indexable="true" name="projectcode" result="false" stat="false" type="ngramtext" xpath="//*[local-name()='entity']/*[local-name()='project']/code"/>
<FIELD indexable="true" name="projectcode_nt" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/code"/>
<FIELD indexable="true" name="projectacronym" result="false" stat="false" type="ngramtext" xpath="//*[local-name()='entity']/*[local-name()='project']/acronym"/>
<FIELD indexable="true" name="projecttitle" result="false" stat="false" type="ngramtext" xpath="//*[local-name()='entity']/*[local-name()='project']/title"/>
<FIELD indexable="true" multivalued="false" name="projectstartdate" result="false" stat="false" type="pdate" value="//*[local-name()='entity']/*[local-name()='project']/startdate"/>
<FIELD indexable="true" multivalued="false" name="projectstartyear" result="false" stat="false" tokenizable="false" value="dnet:extractYear(//*[local-name()='entity']/*[local-name()='project']/startdate)"/>
<FIELD indexable="true" multivalued="false" name="projectenddate" result="false" stat="false" type="pdate" value="//*[local-name()='entity']/*[local-name()='project']/enddate"/>
<FIELD indexable="true" multivalued="false" name="projectendyear" result="false" stat="false" tokenizable="false" value="dnet:extractYear(//*[local-name()='entity']/*[local-name()='project']/enddate)"/>
<FIELD indexable="true" multivalued="false" name="projectcallidentifier" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/callidentifier"/>
<FIELD indexable="true" name="projectkeywords" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='project']/keywords"/>
<FIELD indexable="true" multivalued="false" name="projectduration" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/duration"/>
<FIELD indexable="true" multivalued="false" name="projectecsc39" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='project']/ecsc39)"/>
<FIELD indexable="true" multivalued="false" name="projectoamandatepublications" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/oamandatepublications"/>
<FIELD indexable="true" multivalued="false" name="projectecarticle29_3" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/ecarticle29_3"/>
<FIELD indexable="true" name="projectsubject" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/subjects"/>
<FIELD indexable="true" multivalued="false" name="projectcontracttypename" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/contracttype/@classname"/>
<FIELD indexable="true" name="fundinglevel0_id" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree//funding_level_0/id"/>
<FIELD indexable="true" name="fundinglevel0_name" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree//funding_level_0/name"/>
<FIELD indexable="true" name="fundinglevel0_description" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree//funding_level_0/description"/>
<FIELD indexable="true" name="fundinglevel1_id" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree//funding_level_1/id"/>
<FIELD indexable="true" name="fundinglevel1_name" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree//funding_level_1/name"/>
<FIELD indexable="true" name="fundinglevel1_description" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree//funding_level_1/description"/>
<FIELD indexable="true" name="fundinglevel2_id" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree//funding_level_2/id"/>
<FIELD indexable="true" name="fundinglevel2_name" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree//funding_level_2/name"/>
<FIELD indexable="true" name="fundinglevel2_description" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree//funding_level_2/description"/><!-- PROJECTS' FUNDER FIELDS: indexable only with the new funding path/context handling -->
<FIELD indexable="true" name="funder" result="false" stat="false" tokenizable="false" value="concat(./id/text(), '||', ./name/text(), '||', ./shortname/text())" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree/funder"/>
<FIELD indexable="true" name="fundershortname" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree/funder/shortname"/>
<FIELD indexable="true" name="funderid" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree/funder/id"/>
<FIELD indexable="true" name="fundername" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree/funder/name"/>
<FIELD indexable="true" name="funderoriginalname" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree/funder/originalname"/>
<FIELD indexable="true" name="funderjurisdiction" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='project']/fundingtree/funder/jurisdiction"/><!-- RESULT FIELDS -->
<FIELD indexable="true" name="resulttitle" result="false" stat="false" xpath="//*[local-name() = 'entity']/*[local-name() ='result']/title | //*[local-name()='entity']/*[local-name()='result']/children/result/title"/>
<FIELD indexable="true" name="resultsubject" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/subject)"/>
<FIELD indexable="true" name="resultsubjectclass" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/subject/@classname)"/>
<FIELD indexable="true" multivalued="false" name="resultembargoenddate" result="false" stat="false" type="pdate" value="//*[local-name()='entity']/*[local-name()='result']/embargoenddate"/>
<FIELD indexable="true" multivalued="false" name="resultembargoendyear" result="false" stat="false" tokenizable="false" value="dnet:extractYear(//*[local-name()='entity']/*[local-name()='result']/embargoenddate)"/>
<FIELD indexable="true" multivalued="false" name="resulttypeid" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='result']/resulttype/@classid"/>
<FIELD indexable="true" multivalued="false" name="resulttypename" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='result']/resulttype/@classname"/>
<FIELD indexable="true" multivalued="false" name="resultlanguagename" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='result']/language/@classname"/>
<FIELD indexable="true" name="resultpublisher" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='result']/*[local-name()='publisher']"/>
<FIELD indexable="true" name="resultdescription" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='result']//*[local-name()='description']"/>
<FIELD indexable="true" name="resultlicense" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='result']/children/instance/license"/>
<FIELD indexable="true" name="resultaccessright" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*[local-name()='result']/children/instance/accessright/@classname"/>
<FIELD indexable="true" multivalued="false" name="resultbestaccessright" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/bestaccessright/@classname)"/>
<FIELD indexable="true" multivalued="false" name="resultdateofacceptance" result="false" stat="false" type="pdate" value="//*[local-name()='entity']/*[local-name()='result']/dateofacceptance"/>
<FIELD indexable="true" multivalued="false" name="resultacceptanceyear" result="false" stat="false" tokenizable="false" value="dnet:extractYear(//*[local-name()='entity']/*[local-name()='result']/dateofacceptance)"/>
<FIELD indexable="true" multivalued="true" name="resultauthor" result="false" stat="false" xpath="//*[local-name()='entity']/*[local-name()='result']/creator"/>
<FIELD indexable="true" multivalued="true" name="resultauthor_nt" result="false" stat="false" type="string_ci" xpath="//*[local-name()='entity']/*[local-name()='result']/creator"/>
<FIELD indexable="true" multivalued="true" name="authorid" result="false" stat="false" type="string_ci" xpath="//*[local-name()='entity']/*[local-name()='result']/creator/@*[local-name() != 'rank' and local-name() != 'name' and local-name() != 'surname']"/>
<FIELD indexable="true" multivalued="true" name="authoridtype" result="false" stat="false" type="string_ci" xpath="//*[local-name()='entity']/*[local-name()='result']/creator/@*[local-name() != 'rank' and local-name() != 'name' and local-name() != 'surname']/local-name()"/>
<FIELD indexable="true" name="resulthostingdatasource" result="false" stat="false" tokenizable="false" value="distinct-values(concat(./@id, '||', ./@name))" xpath="//*[local-name()='entity']/*[local-name()='result']/children/instance/*[local-name()='hostedby']"/>
<FIELD indexable="true" name="resulthostingdatasourceid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/children/instance/*[local-name()='hostedby']/@id)"/>
<FIELD indexable="true" name="resulthostingdatasourcename" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/children/instance/*[local-name()='hostedby']/@name)"/>
<FIELD indexable="true" name="instancetypename" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/children/instance/*[local-name()='instancetype']/@classname)"/>
<FIELD indexable="true" name="resultdupid" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*//children/result/@objidentifier"/>
<FIELD indexable="true" name="organizationdupid" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']/*//children/organization/@objidentifier"/>
<FIELD indexable="true" name="externalrefsite" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//children/externalreference/sitename)"/>
<FIELD indexable="true" name="externalreflabel" result="false" stat="false" tokenizable="true" xpath="distinct-values(//*[local-name()='entity']/*//children/externalreference/label)"/>
<FIELD indexable="true" name="externalrefclass" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//children/externalreference/qualifier/@classid)"/>
<FIELD indexable="true" name="externalrefid" result="false" stat="false" tokenizable="false" xpath="(//*[local-name()='entity']/*//children/externalreference/refidentifier)"/>
<FIELD indexable="true" name="resultidentifier" result="false" stat="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/children/instance/webresource/*[local-name()='url'])"/>
<FIELD indexable="true" name="resultsource" result="false" stat="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/source)"/><!-- REL FIELDS -->
<FIELD indexable="true" name="reldatasourcecompatibilityid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel[./to/@type='datasource']/openairecompatibility/@classid)"/>
<FIELD indexable="true" name="relproject" result="false" stat="false" tokenizable="false" value="distinct-values(concat(./text(), '||', dnet:pickFirst(../acronym/text(), ../title/text())))" xpath="//*[local-name()='entity']/*//rel/to[@type='project']"/>
<FIELD indexable="true" name="relprojectid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel/to[@type='project'])"/>
<FIELD indexable="true" name="relprojectcode" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel[./to/@type='project']/code)"/>
<FIELD indexable="true" name="relprojectname" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel[./to/@type='project']/acronym)"/>
<FIELD indexable="true" name="relprojecttitle" result="false" stat="false" xpath="distinct-values(//*[local-name()='entity']/*//rel[./to/@type='project']/title)"/>
<FIELD indexable="true" name="relcontracttypeid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel[./to/@type='project']/contracttype/@classid)"/>
<FIELD indexable="true" name="relcontracttypename" result="false" stat="false" xpath="distinct-values(//*[local-name()='entity']/*//rel[./to/@type='project']/contracttype/@classname)"/>
<FIELD indexable="true" name="relorganizationcountryid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel[./to/@type='organization']/country/@classid)"/>
<FIELD indexable="true" name="relorganizationcountryname" result="false" stat="false" xpath="distinct-values(//*[local-name()='entity']/*//rel[./to/@type='organization']/country/@classname)"/>
<FIELD indexable="true" name="relorganizationid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel/to[@type='organization'])"/>
<FIELD indexable="true" name="relorganizationname" result="false" stat="false" xpath="distinct-values(//*[local-name()='entity']/*//rel[./to/@type='organization']/legalname)"/>
<FIELD indexable="true" name="relorganizationshortname" result="false" stat="false" xpath="distinct-values(//*[local-name()='entity']/*//rel[./to/@type='organization']/legalshortname)"/>
<FIELD indexable="true" name="relresultid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel/to[@type='result'])"/>
<FIELD indexable="true" name="relresulttype" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel/resulttype/@classid)"/>
<FIELD indexable="true" name="relclass" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel/to/@class)"/>
<FIELD indexable="true" name="relfundinglevel0_id" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//rel/funding/funding_level_0"/>
<FIELD indexable="true" name="relfundinglevel0_name" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//rel/funding/funding_level_0/@name/string()"/>
<FIELD indexable="true" name="relfundinglevel1_id" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//rel/funding/funding_level_1"/>
<FIELD indexable="true" name="relfundinglevel1_name" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//rel/funding/funding_level_1/@name/string()"/>
<FIELD indexable="true" name="relfundinglevel2_id" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//rel/funding/funding_level_2"/>
<FIELD indexable="true" name="relfundinglevel2_name" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//rel/funding/funding_level_2/@name/string()"/>
<FIELD indexable="true" name="relinferred" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']//rel/@inferred)"/>
<FIELD indexable="true" name="reltrust" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']//rel/@trust)"/>
<FIELD indexable="true" name="relinferenceprovenance" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']//rel/@inferenceprovenance)"/>
<FIELD indexable="true" name="relprovenanceactionclassid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']//rel/@provenanceaction)"/><!-- PROJECTS' FUNDER FIELDS: indexable only with the new funding path/context handling -->
<FIELD indexable="true" name="relfunder" result="false" stat="false" tokenizable="false" value="distinct-values(concat(@id, '||', @name, '||', @shortname))" xpath="//*[local-name()='entity']//rel/funding/funder"/>
<FIELD indexable="true" name="relfunderid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']//rel/funding/funder/@id)"/>
<FIELD indexable="true" name="relfundershortname" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']//rel/funding/funder/@shortname)"/>
<FIELD indexable="true" name="relfundername" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']//rel/funding/funder/@name)"/>
<FIELD indexable="true" name="relfunderjurisdiction" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']//rel/funding/funder/@jurisdiction)"/><!-- Collected from of the related entity. Available for result-result relationships -->
<FIELD indexable="true" name="relcollectedfromid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel/collectedfrom/@id)"/>
<FIELD indexable="true" name="relcollectedfromname" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*//rel/collectedfrom/@name)"/><!-- COMMON FIELDS -->
<FIELD indexable="true" multivalued="false" name="dateofcollection" result="false" stat="false" type="pdate" value="//header/*[local-name()='dateOfCollection']"/>
<FIELD indexable="true" name="collectedfrom" result="false" stat="false" tokenizable="false" value="distinct-values(concat(./@id, '||', ./@name))" xpath="//*[local-name()='entity']/*/*[local-name()='collectedfrom'] | //*[local-name()='entity']/*//*[local-name() = 'instance']/*[local-name()='collectedfrom']"/>
<FIELD indexable="true" name="collectedfromdatasourceid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*/*[local-name()='collectedfrom']/@id | //*[local-name()='entity']/*//*[local-name() = 'instance']/*[local-name()='collectedfrom']/@id)"/>
<FIELD indexable="true" name="collectedfromname" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*/*[local-name()='collectedfrom']/@name | //*[local-name()='entity']/*//*[local-name() = 'instance']/*[local-name()='collectedfrom']/@name)"/>
<FIELD indexable="true" name="originalid" result="false" stat="false" tokenizable="false" type="string_ci" xpath="//*[local-name()='entity']/*/*[local-name()='originalId']"/>
<FIELD indexable="true" name="pid" result="false" stat="false" tokenizable="false" type="string_ci" xpath="//*[local-name()='entity']/*[local-name()='result']/pid/text()"/>
<FIELD indexable="true" name="pidclassid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/pid/@classid)"/>
<FIELD indexable="true" name="pidclassname" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/pid/@classname)"/>
<FIELD indexable="true" name="inferred" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//datainfo/inferred"/>
<FIELD indexable="true" name="deletedbyinference" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//datainfo/deletedbyinference"/>
<FIELD indexable="true" name="trust" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//datainfo/trust"/>
<FIELD indexable="true" name="inferenceprovenance" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//datainfo/inferenceprovenance"/>
<FIELD indexable="true" name="provenanceactionclassid" result="false" stat="false" tokenizable="false" xpath="//*[local-name()='entity']//datainfo/provenanceaction/@classid"/>
<FIELD indexable="true" name="contextid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/context/@id)"/>
<FIELD indexable="true" name="contexttype" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/context/@type)"/>
<FIELD indexable="true" name="contextname" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/context/@label)"/><!-- Need special fields for community (research initiative) context in order to exclude funders from the context browse -->
<FIELD indexable="true" name="community" result="false" stat="false" tokenizable="false" value="distinct-values(concat(@id, '||', @label))" xpath="//*[local-name()='entity']/*[local-name()='result']/context[@type='community' or @type='ri']"/>
<FIELD indexable="true" name="communityname" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/context[@type='community' or @type='ri']/@label)"/>
<FIELD indexable="true" name="communityid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/context[@type='community' or @type='ri']/@id)"/>
<FIELD indexable="true" name="categoryid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/context/category/@id)"/>
<FIELD indexable="true" name="categoryname" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/context/category/@label)"/>
<FIELD indexable="true" name="conceptid" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/context/category//concept/@id)"/>
<FIELD indexable="true" name="conceptname" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*[local-name()='result']/context/category//concept/@label)"/><!-- new index field for country info from different xpaths for any type of entity -->
<FIELD indexable="true" name="country" result="false" stat="false" tokenizable="false" xpath="distinct-values(//*[local-name()='entity']/*/country/@classid | //*[local-name()='entity']/*//rel[./to/@type='organization']/country/@classid | //*[local-name()='entity']//funder/@jurisdiction)"/><!-- COUNTER FIELDS -->
<FIELD header="true" indexable="true" multivalued="false" name="counter_dedup" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_dedup/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_authorship" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_authorship/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_participation" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_participation/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_similarity" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_similarity/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_publicationdataset" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_publicationDataset/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_publicationdataset_claimed" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_publicationDataset_claimed/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_publicationdataset_collected" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_publicationDataset_collected/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_publicationdataset_inferred" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_publicationDataset_inferred/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_outcome" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_outcome/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_outcome_claimed" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_outcome_claimed/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_outcome_collected" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_outcome_collected/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_outcome_inferred" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_outcome_inferred/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_affiliation" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_affiliation/@value"/>
<FIELD header="true" indexable="true" multivalued="false" name="counter_doi" result="false" stored="true" type="pint" xpath="/record/result/*[local-name()='header']/*[local-name()='counters']/counter_doi/@value"/>
</FIELDS>
</LAYOUT>

94
dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/layoutToRecordTransformer.xsl Normal file
View File

@ -0,0 +1,94 @@
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:nxsl="http://www.w3.org/1999/XSL/TransformXX" version="3.0">
<xsl:output method="xml" omit-xml-declaration="yes"/>
<xsl:namespace-alias result-prefix="xsl" stylesheet-prefix="nxsl"/>
<xsl:param name="format"/>
<xsl:template match="/">
<xsl:apply-templates select="//LAYOUT"/>
</xsl:template>
<xsl:template match="LAYOUT">
<nxsl:stylesheet xmlns:dnet="http://www.d-net.research-infrastructures.eu/saxon-extension" xmlns:dri="http://www.driver-repository.eu/namespace/dri" exclude-result-prefixes="dnet" version="3.0">
<nxsl:output method="xml" omit-xml-declaration="yes" version="1.0"/>
<nxsl:variable name="format">
<xsl:value-of select="$format"/>
</nxsl:variable>
<nxsl:template match="/">
<indexRecord>
<indexRecordIdentifier>
<nxsl:value-of select="//dri:objIdentifier"/>
</indexRecordIdentifier>
<targetFields>
<nxsl:if test="count(//*[local-name()='metadata']/*) &gt; 0">
<xsl:apply-templates select="FIELDS/FIELD[@indexable='true']"/>
</nxsl:if>
</targetFields>
<dnetResult>
<nxsl:copy-of select="/*[local-name()='record']/*[local-name()='result']/*[local-name()='header']"/>
<metadata>
<xsl:apply-templates mode="result" select="FIELDS/FIELD"/>
</metadata>
</dnetResult>
</indexRecord>
</nxsl:template>
</nxsl:stylesheet>
</xsl:template>
<xsl:template match="FIELD[@indexable='true']">
<xsl:choose>
<xsl:when test="@constant">
<xsl:element name="{@name}">
<xsl:value-of select="@constant"/>
</xsl:element>
</xsl:when>
<xsl:when test="@value and not(@xpath)">
<xsl:choose>
<xsl:when test="contains(@type, 'date')">
<nxsl:variable name="{@name}" select="dnet:normalizeDate(normalize-space({normalize-space(@value)}))"/>
<nxsl:if test="string-length(${@name}) &gt; 0">
<nxsl:element name="{@name}">
<nxsl:value-of select="${@name}"/>
</nxsl:element>
</nxsl:if>
</xsl:when>
<xsl:otherwise>
<nxsl:element name="{@name}">
<nxsl:value-of select="{@value}"/>
</nxsl:element>
</xsl:otherwise>
</xsl:choose>
</xsl:when>
<xsl:otherwise>
<xsl:variable name="value">
<xsl:choose>
<xsl:when test="@value">
<xsl:value-of select="@value"/>
</xsl:when>
<xsl:otherwise>
.
</xsl:otherwise>
</xsl:choose>
</xsl:variable>
<nxsl:for-each select="{@xpath}">
<xsl:element name="{@name}">
<xsl:choose>
<xsl:when test="@tokenizable='false'">
<nxsl:value-of select="normalize-space({normalize-space($value)})"/>
</xsl:when>
<xsl:otherwise>
<nxsl:value-of select="{normalize-space($value)}"/>
</xsl:otherwise>
</xsl:choose>
</xsl:element>
</nxsl:for-each>
</xsl:otherwise>
</xsl:choose>
</xsl:template>
<xsl:template match="FIELD" mode="result">
<xsl:if test="@result='true'">
<nxsl:copy-of select="{@xpath}"/>
</xsl:if>
</xsl:template>
<xsl:template match="FIELD" mode="header">
<xsl:if test="@header='true'">
<nxsl:copy-of select="{@xpath}"/>
</xsl:if>
</xsl:template>
</xsl:stylesheet>

750
dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/record.xml Normal file
View File

@ -0,0 +1,750 @@
<result xmlns:dri="http://www.driver-repository.eu/namespace/dri">
<header xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<dri:objIdentifier>dedup_wf_001::113ca964590682d906a3588d3d6b4838</dri:objIdentifier>
<dri:dateOfCollection>2020-03-15T05:46:43.509Z</dri:dateOfCollection>
<dri:dateOfTransformation>2020-03-15T21:17:13.902Z</dri:dateOfTransformation>
</header>
<metadata>
<oaf:entity xmlns:oaf="http://namespace.openaire.eu/oaf"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://namespace.openaire.eu/oaf https://www.openaire.eu/schema/1.0/oaf-1.0.xsd">
<oaf:result>
<collectedfrom name="scholExplorer"
id="openaire____::e034d6a11054f5ade9221ebac484e864"/>
<collectedfrom name="UnpayWall" id="openaire____::8ac8380272269217cb09a928c8caa993"/>
<collectedfrom name="DOAJ-Articles"
id="driver______::bee53aa31dc2cbb538c10c2b65fa5824"/>
<collectedfrom name="Crossref" id="openaire____::081b82f96300b6a6e3d282bad31cb6e2"/>
<collectedfrom name="Microsoft Academic Graph"
id="openaire____::5f532a3fc4f1ea403f37070f59a7a53a"/>
<collectedfrom name="PubMed Central"
id="opendoar____::eda80a3d5b344bc40f3bc04f65b7a357"/>
<originalId>oai:pubmedcentral.nih.gov:5657733</originalId>
<originalId>oai:doaj.org/article:f26495a4c1d149099049e1a604fa1256</originalId>
<originalId>10.1177/0963689717714320</originalId>
<pid classid="pmid" classname="pmid" schemeid="dnet:pid_types"
schemename="dnet:pid_types" inferred="false">28933215</pid>
<pid classid="pmc" classname="pmc" schemeid="dnet:pid_types"
schemename="dnet:pid_types" inferred="false">PMC5657733</pid>
<pid classid="doi" classname="doi" schemeid="dnet:pid_types"
schemename="dnet:pid_types" inferred="false">10.1177/0963689717714320</pid>
<title classid="main title" classname="main title" schemeid="dnet:dataCite_title"
schemename="dnet:dataCite_title" inferred="false">Extract Protects Model Neurons
from Traumatic Injury</title>
<bestaccessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes"
schemename="dnet:access_modes"/>
<creator rank="1">Jain, Subhash C.</creator>
<creator rank="2">Citron, Bruce A.</creator>
<creator rank="3">Vijayalakshmi Ravindranath</creator>
<creator rank="4">Saykally, Jessica N.</creator>
<creator rank="5">Keeley, Kristen L.</creator>
<creator rank="6">Haris Hatic</creator>
<dateofacceptance>2017-06-01</dateofacceptance>
<description>Withania somnifera has been used in traditional medicine for a variety
of neural disorders. Recently, chronic neurodegenerative conditions have been
shown to benefit from treatment with this extract. To evaluate the action of
this extract on traumatically injured neurons, the efficacy of W. somnifera root
extract as a neuroprotective agent was examined in cultured model neurons
exposed to an in vitro injury system designed to mimic mild traumatic brain
injury (TBI). Neuronal health was evaluated by staining with annexin V (an
early, apoptotic feature) and monitoring released lactate dehydrogenase activity
(a terminal cell loss parameter). Potential mechanisms underlying the observed
neuroprotection were examined. Additionally, morphological changes were
monitored following injury and treatment. Although no differences were found in
the expression of the antioxidant transcription factor nuclear factor erythroid
2-like 2 (Nrf2) or other Nrf2-related downstream components, significant changes
were seen in apoptotic signaling. Treatment with the extract resulted in an
increased length of neurites projecting from the neuronal cell body after
injury. W. somnifera extract treatment also resulted in reduced cell death in
the model neuron TBI system. The cell death factor Bax was involved (its
expression was reduced 2-fold by the treatment) and injury-induced reduction in
neurite lengths and numbers was reversed by the treatment. This all indicates
that W. somnifera root extract was neuroprotective and could have therapeutic
potential to target factors involved in secondary injury and long-term sequelae
of mild TBI.</description>
<embargoenddate/>
<subject classid="keyword" classname="keyword"
schemeid="dnet:subject_classification_typologies"
schemename="dnet:subject_classification_typologies" inferred="false">Withania
somnifera</subject>
<subject classid="keyword" classname="keyword"
schemeid="dnet:subject_classification_typologies"
schemename="dnet:subject_classification_typologies" inferred="false">R</subject>
<subject classid="keyword" classname="keyword" schemeid="dnet:subject"
schemename="dnet:subject" inferred="false">Cell Biology</subject>
<subject classid="keyword" classname="keyword"
schemeid="dnet:subject_classification_typologies"
schemename="dnet:subject_classification_typologies" inferred="false"
>neuroprotection</subject>
<subject classid="keyword" classname="keyword"
schemeid="dnet:subject_classification_typologies"
schemename="dnet:subject_classification_typologies" inferred="false"
>SH-SY5Y</subject>
<subject classid="keyword" classname="keyword" schemeid="dnet:subject"
schemename="dnet:subject" inferred="false">Biomedical Engineering</subject>
<subject classid="keyword" classname="keyword" schemeid="dnet:subject"
schemename="dnet:subject" inferred="false">Transplantation</subject>
<subject classid="keyword" classname="keyword"
schemeid="dnet:subject_classification_typologies"
schemename="dnet:subject_classification_typologies" inferred="false">traumatic
brain injury</subject>
<subject classid="keyword" classname="keyword"
schemeid="dnet:subject_classification_typologies"
schemename="dnet:subject_classification_typologies" inferred="false"
>neurites</subject>
<subject classid="keyword" classname="keyword"
schemeid="dnet:subject_classification_typologies"
schemename="dnet:subject_classification_typologies" inferred="false"
>Ayurveda</subject>
<subject classid="keyword" classname="keyword"
schemeid="dnet:subject_classification_typologies"
schemename="dnet:subject_classification_typologies" inferred="false"
>Medicine</subject>
<language classid="eng" classname="English" schemeid="dnet:languages"
schemename="dnet:languages"/>
<relevantdate classid="dnet:date" classname="dnet:date" schemeid="dnet:date"
schemename="dnet:date" inferred="false">2018-11-13</relevantdate>
<relevantdate classid="published-online" classname="published-online"
schemeid="dnet:dataCite_date" schemename="dnet:dataCite_date" inferred="false"
>2017-6-30</relevantdate>
<relevantdate classid="published-print" classname="published-print"
schemeid="dnet:dataCite_date" schemename="dnet:dataCite_date" inferred="false"
>2017-7-1</relevantdate>
<publisher>SAGE Publishing</publisher>
<source>Cell Transplantation, Vol 26 (2017)</source>
<source>Cell Transplantation</source>
<resulttype classid="publication" classname="publication"
schemeid="dnet:result_typologies" schemename="dnet:result_typologies"/>
<resourcetype/>
<journal issn="0963-6897" eissn="1555-3892" ep="1201" iss="7" sp="1193" vol="26"
>Cell Transplantation</journal>
<context id="NIH" label="National Institutes of Health" type="funding">
<category id="NIH::VETERANS AFFAIRS" label="VETERANS AFFAIRS"/>
</context>
<datainfo>
<inferred>true</inferred>
<deletedbyinference>false</deletedbyinference>
<trust>0.9</trust>
<inferenceprovenance>dedup-similarity-result-levenstein</inferenceprovenance>
<provenanceaction classid="sysimport:dedup" classname="sysimport:dedup"
schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
</datainfo>
<rels>
<rel inferred="true" trust="0.8998"
inferenceprovenance="iis::document_affiliations" provenanceaction="iis">
<to class="hasAuthorInstitution"
scheme="dnet:organization_organization_relations" type="organization"
>wt__________::4de25ac59f6cb729d5716260164bb67c</to>
<legalname>Indian Institute Of Science</legalname>
</rel>
<rel inferred="true" trust="0.8998"
inferenceprovenance="iis::document_affiliations" provenanceaction="iis">
<to class="hasAuthorInstitution"
scheme="dnet:organization_organization_relations" type="organization"
>nih_________::ba7da8316fd53d04a985bc935e438555</to>
<legalname>INDIAN INSTITUTE OF SCIENCE</legalname>
</rel>
<rel inferred="true" trust="0.9" inferenceprovenance=""
provenanceaction="sysimport:actionset">
<to class="hasAuthorInstitution"
scheme="dnet:organization_organization_relations" type="organization"
>dedup_wf_001::0047940c0207b6a83e79cd803ecf17d1</to>
<country classid="GB" classname="United Kingdom" schemeid="dnet:countries"
schemename="dnet:countries"/>
<legalname>MRC - MRC Laboratory of Molecular Biology</legalname>
<legalshortname>LMB</legalshortname>
</rel>
<rel inferred="true" trust="0.8847"
inferenceprovenance="iis::document_affiliations" provenanceaction="iis">
<to class="hasAuthorInstitution"
scheme="dnet:organization_organization_relations" type="organization"
>rcuk________::2558c4f3132f6907f7b23c69009f0d87</to>
<legalname>INDIAN INSTUTUTE OF SCIENCE</legalname>
</rel>
<rel inferred="true" trust="0.9" inferenceprovenance=""
provenanceaction="sysimport:actionset">
<to class="hasAuthorInstitution"
scheme="dnet:organization_organization_relations" type="organization"
>dedup_wf_001::d2fdc8e80f8b4365091bcea83f918ccf</to>
<country classid="IN" classname="India" schemeid="dnet:countries"
schemename="dnet:countries"/>
<legalname>University of Delhi</legalname>
<legalshortname>University of Delhi</legalshortname>
</rel>
<rel inferred="true" trust="0.7200000000000001"
inferenceprovenance="dedup-similarity-result-levenstein" provenanceaction="">
<to class="merges" scheme="dnet:result_result_relations" type="publication"
>doiboost____::d5177e3ad00bd9288201b60206a0b5d0</to>
<dateofacceptance>2017-6-30</dateofacceptance>
<collectedfrom name="Crossref"
id="openaire____::081b82f96300b6a6e3d282bad31cb6e2"/>
<collectedfrom name="UnpayWall"
id="openaire____::8ac8380272269217cb09a928c8caa993"/>
<collectedfrom name="Microsoft Academic Graph"
id="openaire____::5f532a3fc4f1ea403f37070f59a7a53a"/>
<pid classid="doi" classname="doi" schemeid="dnet:pid_types"
schemename="dnet:pid_types" inferred="false"
>10.1177/0963689717714320</pid>
</rel>
<rel inferred="true" trust="0.7200000000000001"
inferenceprovenance="dedup-similarity-result-levenstein" provenanceaction="">
<to class="merges" scheme="dnet:result_result_relations" type="publication"
>od_______267::fb470352a4b33af7c83391c02117c4fc</to>
<collectedfrom name="PubMed Central"
id="opendoar____::eda80a3d5b344bc40f3bc04f65b7a357"/>
<publisher>SAGE Publications</publisher>
<pid classid="pmc" classname="pmc" schemeid="dnet:pid_types"
schemename="dnet:pid_types" inferred="false">PMC5657733</pid>
<pid classid="pmid" classname="pmid" schemeid="dnet:pid_types"
schemename="dnet:pid_types" inferred="false">28933215</pid>
<dateofacceptance>2017-06-01</dateofacceptance>
<pid classid="doi" classname="doi" schemeid="dnet:pid_types"
schemename="dnet:pid_types" inferred="false"
>10.1177/0963689717714320</pid>
</rel>
<rel inferred="true" trust="0.6249"
inferenceprovenance="iis::document_referencedProjects"
provenanceaction="iis">
<to class="isProducedBy" scheme="dnet:project_project_relations"
type="project">nih_________::24e81ae35bbcb50c778df1039f912617</to>
<funding>
<funder id="nih_________::NIH" shortname="NIH"
name="National Institutes of Health" jurisdiction="US"/>
<funding_level_0 name="VETERANS AFFAIRS"
>nih_________::NIH::VETERANS_AFFAIRS</funding_level_0>
</funding>
<title>Preventing TBI-Induced Chronic Functional Loss with a Neuroprotective
Antioxidant</title>
<code>1I01RX001520-01A1</code>
</rel>
<rel inferred="true" trust="0.8998"
inferenceprovenance="iis::document_affiliations" provenanceaction="iis">
<to class="hasAuthorInstitution"
scheme="dnet:organization_organization_relations" type="organization"
>wt__________::52e59d4aa1c57bda1ec144f409de83fc</to>
<legalname>Indian Institute of Science</legalname>
</rel>
<rel inferred="true" trust="0.8998"
inferenceprovenance="iis::document_affiliations" provenanceaction="iis">
<to class="hasAuthorInstitution"
scheme="dnet:organization_organization_relations" type="organization"
>dedup_wf_001::0499ff413ba8e7fa686531725ba12338</to>
<legalshortname>IISc</legalshortname>
<country classid="IN" classname="India" schemeid="dnet:countries"
schemename="dnet:countries"/>
<legalname>Indian Institute of Science</legalname>
</rel>
<rel inferred="true" trust="0.8998"
inferenceprovenance="iis::document_affiliations" provenanceaction="iis">
<to class="hasAuthorInstitution"
scheme="dnet:organization_organization_relations" type="organization"
>wt__________::ba1db3669859a46e72f222052a9a26d8</to>
<legalname>University of Delhi</legalname>
</rel>
<rel inferred="true" trust="0.9" inferenceprovenance=""
provenanceaction="sysimport:actionset">
<to class="hasAuthorInstitution"
scheme="dnet:organization_organization_relations" type="organization"
>dedup_wf_001::17c785347dfb060aa115af824b0c6789</to>
<legalshortname>IISc</legalshortname>
<country classid="IN" classname="India" schemeid="dnet:countries"
schemename="dnet:countries"/>
<legalname>Indian Institute of Science Bangalore</legalname>
</rel>
<rel inferred="true" trust="0.7200000000000001"
inferenceprovenance="dedup-similarity-result-levenstein" provenanceaction="">
<to class="merges" scheme="dnet:result_result_relations" type="publication"
>scholexplore::16181ec1a2484116e8ed6b3348858fe7</to>
<collectedfrom name="scholExplorer"
id="openaire____::e034d6a11054f5ade9221ebac484e864"/>
<pid classid="pmid" classname="pmid" schemeid="dnet:pid_types"
schemename="dnet:pid_types" inferred="false">28933215</pid>
</rel>
<rel inferred="true" trust="0.7200000000000001"
inferenceprovenance="dedup-similarity-result-levenstein" provenanceaction="">
<to class="merges" scheme="dnet:result_result_relations" type="publication"
>doajarticles::cac994ec6c322070c41474486eb5c595</to>
<dateofacceptance>2017-07-01</dateofacceptance>
<collectedfrom name="DOAJ-Articles"
id="driver______::bee53aa31dc2cbb538c10c2b65fa5824"/>
<publisher>SAGE Publishing</publisher>
<pid classid="doi" classname="doi" schemeid="dnet:pid_types"
schemename="dnet:pid_types" inferred="false"
>10.1177/0963689717714320</pid>
</rel>
<rel inferred="true" trust="0.6775"
inferenceprovenance="iis::document_referencedDatasets"
provenanceaction="iis">
<to class="isRelatedTo" scheme="dnet:result_result_relations" type="dataset"
>r37980778c78::39a72c53d5801325784f728b543a49a1</to>
<collectedfrom name="figshare"
id="re3data_____::7980778c78fb4cf0fab13ce2159030dc"/>
<pid classid="doi" classname="doi" schemeid="dnet:pid_types"
schemename="dnet:pid_types" inferred="false"
>10.1371/journal.pone.0006628</pid>
<dateofacceptance>2016-01-01</dateofacceptance>
<publisher>Figshare</publisher>
</rel>
<rel inferred="true" trust="0.8998"
inferenceprovenance="iis::document_affiliations" provenanceaction="iis">
<to class="hasAuthorInstitution"
scheme="dnet:organization_organization_relations" type="organization"
>rcuk________::23feba2a5ca7f6b6016bf3a45180da50</to>
<legalname>University of Delhi</legalname>
</rel>
</rels>
<children>
<instance id="openaire____::55045bd2a65019fd8e6741a755395c8c">
<accessright classid="UNKNOWN" classname="not available"
schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
<collectedfrom name="scholExplorer"
id="openaire____::e034d6a11054f5ade9221ebac484e864"/>
<hostedby name="Unknown Repository"
id="openaire____::55045bd2a65019fd8e6741a755395c8c"/>
<instancetype classid="0000" classname="Unknown"
schemeid="dnet:publication_resource"
schemename="dnet:publication_resource"/>
<webresource>
<url>https://www.ncbi.nlm.nih.gov/pubmed/28933215</url>
</webresource>
</instance>
<instance id="opendoar____::8b6dd7db9af49e67306feb59a8bdc52c">
<accessright classid="OPEN" classname="Open Access"
schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
<collectedfrom name="PubMed Central"
id="opendoar____::eda80a3d5b344bc40f3bc04f65b7a357"/>
<hostedby name="Europe PubMed Central"
id="opendoar____::8b6dd7db9af49e67306feb59a8bdc52c"/>
<dateofacceptance>2017-06-01</dateofacceptance>
<instancetype classid="0001" classname="Article"
schemeid="dnet:publication_resource"
schemename="dnet:publication_resource"/>
<webresource>
<url>http://europepmc.org/articles/PMC5657733</url>
</webresource>
</instance>
<instance id="openaire____::55045bd2a65019fd8e6741a755395c8c">
<accessright classid="UNKNOWN" classname="not available"
schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
<collectedfrom name="Crossref"
id="openaire____::081b82f96300b6a6e3d282bad31cb6e2"/>
<hostedby name="Unknown Repository"
id="openaire____::55045bd2a65019fd8e6741a755395c8c"/>
<instancetype classid="0001" classname="Article"
schemeid="dnet:publication_resource"
schemename="dnet:publication_resource"/>
<webresource>
<url>http://journals.sagepub.com/doi/full-xml/10.1177/0963689717714320</url>
</webresource>
<webresource>
<url>http://journals.sagepub.com/doi/pdf/10.1177/0963689717714320</url>
</webresource>
<webresource>
<url>https://academic.microsoft.com/#/detail/2588640354</url>
</webresource>
</instance>
<instance id="doajarticles::4607792f6beaa6e33d5ff5c63830da98">
<accessright classid="OPEN" classname="Open Access"
schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
<collectedfrom name="DOAJ-Articles"
id="driver______::bee53aa31dc2cbb538c10c2b65fa5824"/>
<hostedby name="Cell Transplantation"
id="doajarticles::4607792f6beaa6e33d5ff5c63830da98"/>
<dateofacceptance>2017-07-01</dateofacceptance>
<instancetype classid="0001" classname="Article"
schemeid="dnet:publication_resource"
schemename="dnet:publication_resource"/>
<webresource>
<url>https://doi.org/10.1177/0963689717714320</url>
</webresource>
<webresource>
<url>https://doaj.org/toc/0963-6897</url>
</webresource>
<webresource>
<url>https://doaj.org/toc/1555-3892</url>
</webresource>
</instance>
<instance id="doajarticles::4607792f6beaa6e33d5ff5c63830da98">
<accessright classid="RESTRICTED" classname="Restricted"
schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
<collectedfrom name="Crossref"
id="openaire____::081b82f96300b6a6e3d282bad31cb6e2"/>
<hostedby name="Cell Transplantation"
id="doajarticles::4607792f6beaa6e33d5ff5c63830da98"/>
<instancetype classid="0001" classname="Article"
schemeid="dnet:publication_resource"
schemename="dnet:publication_resource"/>
<webresource>
<url>http://dx.doi.org/10.1177/0963689717714320</url>
</webresource>
</instance>
<instance id="openaire____::55045bd2a65019fd8e6741a755395c8c">
<accessright classid="OPEN" classname="Open Access"
schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
<collectedfrom name="UnpayWall"
id="openaire____::8ac8380272269217cb09a928c8caa993"/>
<hostedby name="Unknown Repository"
id="openaire____::55045bd2a65019fd8e6741a755395c8c"/>
<instancetype classid="0001" classname="Article"
schemeid="dnet:publication_resource"
schemename="dnet:publication_resource"/>
<webresource>
<url>https://journals.sagepub.com/doi/pdf/10.1177/0963689717714320</url>
</webresource>
</instance>
</children>
</oaf:result>
<extraInfo name="result citations" typology="citations"
provenance="iis::document_referencedDocuments" trust="0.9">
<citations>
<citation position="1">
<rawText>1 Bryan-Hancock C Harrison J The global burden of traumatic brain
injury: preliminary results from the Global Burden of Disease Project.
Inj Prev. 2010;16(Suppl 1):A17.</rawText>
</citation>
<citation position="2">
<rawText>2 Gardner RC Yaffe K Epidemiology of mild traumatic brain injury
and neurodegenerative disease. Mol Cell Neurosci. 2015;66(Pt
B):75–80.25748121</rawText>
<id value="dedup_wf_001::7ea5a74d23841025462422c95f25f5de" type="openaire"
confidenceLevel="0.9"/>
<id value="25748121" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="3">
<rawText>3 Stern RA Riley DO Daneshvar DH Nowinski CJ Cantu RC McKee AC
Long-term consequences of repetitive brain trauma: chronic traumatic
encephalopathy. PM R. 2011;3(10 Suppl 2):S460–S467.22035690</rawText>
<id value="scholexplore::4cd90a17b23dffce9a8cf58d81ca9e31" type="openaire"
confidenceLevel="0.9"/>
<id value="22035690" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="4">
<rawText>4 Mac Donald CL Johnson AM Cooper D Nelson EC Werner NJ Shimony JS
Snyder AZ Raichle ME Witherow JR Fang R Detection of blast-related
traumatic brain injury in U.S. military personnel. N Engl J Med.
2011;364(22):2091–2100.21631321</rawText>
<id value="dedup_wf_001::81f1d0d6f713d335cb4f8dbbd5aea3b5" type="openaire"
confidenceLevel="0.9"/>
<id value="21631321" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="5">
<rawText>5 Hatic H Kane MJ Saykally JN Citron BA Modulation of transcription
factor Nrf2 in an in vitro model of traumatic brain injury. J
Neurotrauma. 2012;29(6):1188–1196.22201269</rawText>
<id value="scholexplore::5a883cd3f6eab6dcfd9253dc3a667794" type="openaire"
confidenceLevel="0.9"/>
<id value="22201269" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="6">
<rawText>6 Saykally JN Rachmany L Hatic H Shaer A Rubovitch V Pick CG Citron
BA The nuclear factor erythroid 2-like 2 activator,
tert-butylhydroquinone, improves cognitive performance in mice after
mild traumatic brain injury. Neuroscience.
2012;223:305–314.22890082</rawText>
<id value="dedup_wf_001::e1fdece0d7d8e81cf0497971758a07d7" type="openaire"
confidenceLevel="0.9"/>
<id value="22890082" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="7">
<rawText>7 Hall ED Vaishnav RA Mustafa AG Antioxidant therapies for
traumatic brain injury. Neurotherapeutics.
2010;7(1):51–61.20129497</rawText>
<id value="dedup_wf_001::dad85993cf85ecab6c20834b36da8fde" type="openaire"
confidenceLevel="0.9"/>
<id value="20129497" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="8">
<rawText>8 Scartezzini P Speroni E Review on some plants of Indian
traditional medicine with antioxidant activity. J Ethnopharmacol.
2000;71(1–2):23–43.10904144</rawText>
<id value="dedup_wf_001::4bdc528dcf25cfa21d6d668263b519f4" type="openaire"
confidenceLevel="0.9"/>
<id value="10904144" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="9">
<rawText>9 Mishra LC Singh BB Dagenais S Scientific basis for the
therapeutic use of Withania somnifera (ashwagandha): a review. Altern
Med Rev. 2000;5(4):334–346.10956379</rawText>
<id value="scholexplore::8093622f33e4a82bf55d0dc741b5023c" type="openaire"
confidenceLevel="0.9"/>
<id value="10956379" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="10">
<rawText>10 Singh RH Exploring larger evidence-base for contemporary
Ayurveda. Int J Ayurveda Res. 2010;1(2):65–66.20814517</rawText>
<id value="dedup_wf_001::ceac054eb7c6b9bd15dfad24c6e339a5" type="openaire"
confidenceLevel="0.9"/>
<id value="20814517" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="11">
<rawText>11 Alam N Hossain M Mottalib MA Sulaiman SA Gan SH Khalil MI
Methanolic extracts of Withania somnifera leaves, fruits and roots
possess antioxidant properties and antibacterial activities. BMC
Complement Altern Med. 2012;12:175.23039061</rawText>
<id value="dedup_wf_001::d85dd0b9c21a5859b53d45391bc905c8" type="openaire"
confidenceLevel="0.9"/>
<id value="23039061" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="12">
<rawText>12 Gupta GL Rana AC Withania somnifera (ashwagandha): a review.
Pharmacognosy Rev. 2007;1(1):129–136.</rawText>
</citation>
<citation position="13">
<rawText>13 Durg S Dhadde SB Vandal R Shivakumar BS Charan CS Withania
somnifera (ashwagandha) in neurobehavioural disorders induced by brain
oxidative stress in rodents: a systematic review and meta-analysis. J
Pharm Pharmacol. 2015;67(7):879–899.25828061</rawText>
<id value="dedup_wf_001::aec7c43727fca72e466703fad9cd50d0" type="openaire"
confidenceLevel="0.9"/>
<id value="25828061" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="14">
<rawText>14 Kuboyama T Tohda C Komatsu K Effects of ashwagandha (roots of
Withania somnifera) on neurodegenerative diseases. Biol Pharm Bull.
2014;37(6):892–897.24882401</rawText>
<id value="dedup_wf_001::e475da1c365be8b7938bd439cd91b1ce" type="openaire"
confidenceLevel="0.9"/>
<id value="24882401" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="15">
<rawText>15 Mirjalili MH Moyano E Bonfill M Cusido RM Palazon J Steroidal
lactones from Withania somnifera, an ancient plant for novel medicine.
Molecules. 2009;14(7):2373–2393.19633611</rawText>
<id value="dedup_wf_001::034ea95279138fb4f2e3fc41e50c953f" type="openaire"
confidenceLevel="0.9"/>
<id value="19633611" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="16">
<rawText>16 Ven Murthy MR Ranjekar PK Ramassamy C Deshpande M Scientific
basis for the use of Indian ayurvedic medicinal plants in the treatment
of neurodegenerative disorders: ashwagandha. Cent Nerv Syst Agents Med
Chem. 2010;10(3):238–246.20528765</rawText>
<id value="scholexplore::ec8cea63a81d526a1a8133af3557674f" type="openaire"
confidenceLevel="0.9"/>
<id value="20528765" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="17">
<rawText>17 Singh RH Narsimhamurthy K Singh G Neuronutrient impact of
Ayurvedic Rasayana therapy in brain aging. Biogerontology.
2008;9(6):369–374.18931935</rawText>
<id value="dedup_wf_001::41e1d24589d4c1e6b7bf322732b69227" type="openaire"
confidenceLevel="0.9"/>
<id value="18931935" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="18">
<rawText>18 Kulkarni SK Dhir A Withania somnifera: an Indian ginseng. Prog
Neuropsychopharmacol Biol Psychiatry.
2008;32(5):1093–1105.17959291</rawText>
<id value="dedup_wf_001::c0c07dd044ffb2996b047e09c0eb8891" type="openaire"
confidenceLevel="0.9"/>
<id value="17959291" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="19">
<rawText>19 Cooley K Szczurko O Perri D Mills EJ Bernhardt B Zhou Q Seely D
Naturopathic care for anxiety: a randomized controlled trial
ISRCTN78958974. PLoS One. 2009;4(8):e6628.19718255</rawText>
<id value="dedup_wf_001::6c8ad2b3098d9bf2207317744f8c8632" type="openaire"
confidenceLevel="0.9"/>
<id value="19718255" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="20">
<rawText>20 Chopra A Lavin P Patwardhan B Chitre D A 32-week randomized,
placebo-controlled clinical evaluation of RA-11, an Ayurvedic drug, on
osteoarthritis of the knees. J Clin Rheumatol.
2004;10(5):236–245.17043520</rawText>
<id value="dedup_wf_001::6ad0c8932a7eab3967c0800822af5177" type="openaire"
confidenceLevel="0.9"/>
<id value="17043520" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="21">
<rawText>21 Chaudhary G Sharma U Jagannathan NR Gupta YK Evaluation of
Withania somnifera in a middle cerebral artery occlusion model of stroke
in rats. Clin Exp Pharmacol Physiol.
2003;30(5–6):399–404.12859433</rawText>
<id value="12859433" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="22">
<rawText>22 Adams JD Jr Yang J Mishra LC Singh BB Effects of ashwagandha in
a rat model of stroke. Altern Ther Health Med.
2002;8(5):18–19.</rawText>
</citation>
<citation position="23">
<rawText>23 Baitharu I Jain V Deep SN Hota KB Hota SK Prasad D Ilavazhagan G
Withania somnifera root extract ameliorates hypobaric hypoxia induced
memory impairment in rats. J Ethnopharmacol.
2013;145(2):431–441.23211660</rawText>
<id value="dedup_wf_001::744d9bbe76384c0e933cb2b8099105ba" type="openaire"
confidenceLevel="0.9"/>
<id value="23211660" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="24">
<rawText>24 RajaSankar S Manivasagam T Sankar V Prakash S Muthusamy R
Krishnamurti A Surendran S Withania somnifera root extract improves
catecholamines and physiological abnormalities seen in a Parkinson’s
disease model mouse. J Ethnopharmacol.
2009;125(3):369–373.19666100</rawText>
<id value="dedup_wf_001::7d60b85f60390db7b841f13778fbbd8a" type="openaire"
confidenceLevel="0.9"/>
<id value="19666100" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="25">
<rawText>25 Pingali U Pilli R Fatima N Effect of standardized aqueous
extract of Withania somnifera on tests of cognitive and psychomotor
performanc e in healthy human participants. Pharmacognosy Res.
2014;6(1):12–18.24497737</rawText>
<id value="dedup_wf_001::024f29db87472548b39ca9ce8ff4c76e" type="openaire"
confidenceLevel="0.9"/>
<id value="24497737" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="26">
<rawText>26 Prabhakaran Y Dinakaran SK Macharala SP Ghosh S Karanam SR
Kanthasamy N Avasarala H Molecular docking studies of withanolides
against Cox-2 enzyme. Pak J Pharm Sci.
2012;25(3):595–598.22713947</rawText>
<id value="dedup_wf_001::5ff38da7b7b422ff22180741a908aa14" type="openaire"
confidenceLevel="0.9"/>
<id value="22713947" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="27">
<rawText>27 Mohan R Hammers HJ Bargagna-Mohan P Zhan XH Herbstritt CJ Ruiz A
Zhang L Hanson AD Conner BP Rougas J Withaferin A is a potent inhibitor
of angiogenesis. Angiogenesis. 2004;7(2):115–122.15516832</rawText>
<id value="dedup_wf_001::dd552ab21a101677856a159c9d74908a" type="openaire"
confidenceLevel="0.9"/>
<id value="15516832" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="28">
<rawText>28 Friedemann T Otto B Klatschke K Schumacher U Tao Y Leung AK
Efferth T Schroder S Coptis chinensis Franch. exhibits neuroprotective
properties against oxidative stress in human neuroblastoma cells. J
Ethnopharmacol. 2014;155(1):607–615.24929105</rawText>
<id value="dedup_wf_001::9f8a8f6e0aedf9dff47a66b8229baf87" type="openaire"
confidenceLevel="0.9"/>
<id value="24929105" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="29">
<rawText>29 Hu S Han R Mak S Han Y Protection against
1-methyl-4-phenylpyridinium ion (MPP+)-induced apoptosis by water
extract of ginseng (Panax ginseng C.A. Meyer) in SH-SY5Y cells. J
Ethnopharmacol. 2011;135(1):34–42.21349320</rawText>
<id value="scholexplore::646e892807dc6e1a30938d439b879a76" type="openaire"
confidenceLevel="0.9"/>
<id value="21349320" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="30">
<rawText>30 Kane MJ Hatic H Delic V Dennis JS Butler CL Saykally JN Citron
BA Modeling the pathobiology of repetitive traumatic brain injury in
immortalized neuronal cell lines. Brain Res.
2011;1425:123–131.22018688</rawText>
<id value="22018688" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="31">
<rawText>31 Sehgal N Gupta A Valli RK Joshi SD Mills JT Hamel E Khanna P
Jain SC Thakur SS Ravindranath V Withania somnifera reverses Alzheimer’s
disease pathology by enhancing low-density lipoprotein receptor-related
protein in liver. Proc Natl Acad Sci U S A.
2012;109(9):3510–3515.22308347</rawText>
<id value="dedup_wf_001::6a5a977979d2b9beed791c95191ca124" type="openaire"
confidenceLevel="0.9"/>
<id value="22308347" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="32">
<rawText>32 Arundine M Aarts M Lau A Tymianski M Vulnerability of central
neurons to secondary insults after in vitro mechanical stretch. J
Neurosci. 2004;24(37):8106–8123.15371512</rawText>
<id value="scholexplore::a92bd7b567b76e9a41c190ee141483da" type="openaire"
confidenceLevel="0.9"/>
<id value="15371512" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="33">
<rawText>33 Lau A Arundine M Sun HS Jones M Tymianski M Inhibition of
caspase-mediated apoptosis by peroxynitrite in traumatic brain injury. J
Neurosci. 2006;26(45):11540–11553.17093075</rawText>
<id value="17093075" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="34">
<rawText>34 Weber JT Rzigalinski BA Ellis EF Traumatic injury of cortical
neurons causes changes in intracellular calcium stores and capacitative
calcium influx. J Biol Chem. 2001;276(3):1800–1807.11050103</rawText>
<id value="11050103" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="35">
<rawText>35 Ellis EF McKinney JS Willoughby KA Liang S Povlishock JT A new
model for rapid stretch-induced injury of cells in culture:
characterization of the model using astrocytes. J Neurotrauma.
1995;12(3):325–339.7473807</rawText>
<id value="7473807" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="36">
<rawText>36 Zhang Y Ba Y Liu C Sun G Ding L Gao S Hao J Yu Z Zhang J Zen K
PGC-1alpha induces apoptosis in human epithelial ovarian cancer cells
through a PPARgamma-dependent pathway. Cell Res.
2007;17(4):363–373.17372612</rawText>
<id value="17372612" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="37">
<rawText>37 Brooks AR Lelkes PI Rubanyi GM Gene expression profiling of
human aortic endothelial cells exposed to disturbed flow and steady
laminar flow. Physiol Genomics. 2002;9(1):27–41.11948288</rawText>
<id value="11948288" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="38">
<rawText>38 Du Y Villeneuve NF Wang XJ Sun Z Chen W Li J Lou H Wong PK Zhang
DD Oridonin confers protection against arsenic-induced toxicity through
activation of the Nrf2-mediated defensive response. Environ Health
Perspect. 2008;116(9):1154–1161.18795156</rawText>
<id value="dedup_wf_001::c4cf0ed4f97b1c6e4c75fcb9c7afe9e0" type="openaire"
confidenceLevel="0.9"/>
<id value="18795156" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="39">
<rawText>39 Pool M Thiemann J Bar-Or A Fournier AE NeuriteTracer: a novel
ImageJ plugin for automated quantification of neurite outgrowth. J
Neurosci Methods. 2008;168(1):134–139.17936365</rawText>
<id value="dedup_wf_001::6880dcaf7780c1e1c331a14f57657cb9" type="openaire"
confidenceLevel="0.9"/>
<id value="17936365" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="40">
<rawText>40 Chen J Wu X Shao B Zhao W Shi W Zhang S Ni L Shen A Increased
expression of TNF receptor-associated factor 6 after rat traumatic brain
injury. Cell Mol Neurobiol. 2011;31(2):269–275.21072581</rawText>
<id value="21072581" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="41">
<rawText>41 Kuboyama T Tohda C Komatsu K Neuritic regeneration and synaptic
reconstruction induced by withanolide A. Br J Pharmacol.
2005;144(7):961–971.15711595</rawText>
<id value="scholexplore::b4df67f5a2958ef86b5ad05198145918" type="openaire"
confidenceLevel="0.9"/>
<id value="15711595" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="42">
<rawText>42 Kuboyama T Tohda C Komatsu K Withanoside IV and its active
metabolite, sominone, attenuate Abeta(25-35)-induced neurodegeneration
Eur J Neurosci. 2006;23(6):1417–1426.16553605</rawText>
<id value="scholexplore::76a1d92beacaba8b41d27bcfe74b499d" type="openaire"
confidenceLevel="0.9"/>
<id value="16553605" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="43">
<rawText>43 Jarrard LE On the role of the hippocampus in learning and memory
in the rat. Behav Neural Biol. 1993;60(1):9–26.8216164</rawText>
<id value="dedup_wf_001::b3c0dbd186731e31ce56da1154570e0f" type="openaire"
confidenceLevel="0.9"/>
<id value="8216164" type="pmid" confidenceLevel="0.9"/>
</citation>
<citation position="44">
<rawText>44 Vareed SK Bauer AK Nair KM Liu Y Jayaprakasam B Nair MG
Blood-brain barrier permeability of bioactive withanamides present in
Withania somnifera fruit extract. Phytother Res.
2014;28(8):1260–1264.24458838</rawText>
<id value="scholexplore::72d81005d8571c5fd59daf66a9d0fc73" type="openaire"
confidenceLevel="0.9"/>
<id value="24458838" type="pmid" confidenceLevel="0.9"/>
</citation>
</citations>
</extraInfo>
</oaf:entity>
</metadata>
</result>

6
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/config-default.xml
View File

@ -15,12 +15,12 @@
<name>oozie.action.sharelib.for.spark</name> <name>oozie.action.sharelib.for.spark</name>
<value>spark2</value> <value>spark2</value>
</property> </property>
<property> <property>
<name>hive_metastore_uris</name> <name>hiveMetastoreUris</name>
<value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value> <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
</property> </property>
<property> <property>
<name>hive_jdbc_url</name> <name>hiveJdbcUrl</name>
<value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value> <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
</property> </property>
<property> <property>

166
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -14,11 +14,11 @@
<description>the external stats that should be added since they are not included in the graph database</description> <description>the external stats that should be added since they are not included in the graph database</description>
</property> </property>
<property> <property>
<name>hive_metastore_uris</name> <name>hiveMetastoreUris</name>
<description>hive server metastore URIs</description> <description>hive server metastore URIs</description>
</property> </property>
<property> <property>
<name>hive_jdbc_url</name> <name>hiveJdbcUrl</name>
<description>hive server jdbc url</description> <description>hive server jdbc url</description>
</property> </property>
</parameters> </parameters>
@ -29,7 +29,7 @@
<configuration> <configuration>
<property> <property>
<name>hive.metastore.uris</name> <name>hive.metastore.uris</name>
<value>${hive_metastore_uris}</value> <value>${hiveMetastoreUris}</value>
</property> </property>
</configuration> </configuration>
</global> </global>
@ -42,7 +42,7 @@
<action name="Step1"> <action name="Step1">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step1.sql</script> <script>scripts/step1.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -53,7 +53,7 @@
<action name="Step2_1"> <action name="Step2_1">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step2_1.sql</script> <script>scripts/step2_1.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -64,7 +64,7 @@
<action name="Step2_2"> <action name="Step2_2">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step2_2.sql</script> <script>scripts/step2_2.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -75,7 +75,7 @@
<action name="Step2_3"> <action name="Step2_3">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step2_3.sql</script> <script>scripts/step2_3.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -86,7 +86,7 @@
<action name="Step2_4"> <action name="Step2_4">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step2_4.sql</script> <script>scripts/step2_4.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -97,7 +97,7 @@
<action name="Step2_5"> <action name="Step2_5">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step2_5.sql</script> <script>scripts/step2_5.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -108,7 +108,7 @@
<action name="Step2_6"> <action name="Step2_6">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step2_6.sql</script> <script>scripts/step2_6.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -119,7 +119,7 @@
<action name="Step2_7"> <action name="Step2_7">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step2_7.sql</script> <script>scripts/step2_7.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -130,7 +130,7 @@
<action name="Step2_8"> <action name="Step2_8">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step2_8.sql</script> <script>scripts/step2_8.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -141,7 +141,7 @@
<action name="Step2_9"> <action name="Step2_9">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step2_9.sql</script> <script>scripts/step2_9.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -152,7 +152,7 @@
<action name="Step3"> <action name="Step3">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step3.sql</script> <script>scripts/step3.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -163,7 +163,7 @@
<action name="Step4_1"> <action name="Step4_1">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step4_1.sql</script> <script>scripts/step4_1.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -174,7 +174,7 @@
<action name="Step4_2"> <action name="Step4_2">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step4_2.sql</script> <script>scripts/step4_2.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -185,7 +185,7 @@
<action name="Step4_3"> <action name="Step4_3">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step4_3.sql</script> <script>scripts/step4_3.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -196,7 +196,7 @@
<action name="Step4_4"> <action name="Step4_4">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step4_4.sql</script> <script>scripts/step4_4.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -207,7 +207,7 @@
<action name="Step4_5"> <action name="Step4_5">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step4_5.sql</script> <script>scripts/step4_5.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -218,7 +218,7 @@
<action name="Step4_6"> <action name="Step4_6">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step4_6.sql</script> <script>scripts/step4_6.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -229,7 +229,7 @@
<action name="Step4_7"> <action name="Step4_7">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step4_7.sql</script> <script>scripts/step4_7.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -240,7 +240,7 @@
<action name="Step4_8"> <action name="Step4_8">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step4_8.sql</script> <script>scripts/step4_8.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -251,7 +251,7 @@
<action name="Step4_9"> <action name="Step4_9">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step4_9.sql</script> <script>scripts/step4_9.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -262,7 +262,7 @@
<action name="Step4_10"> <action name="Step4_10">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step4_10.sql</script> <script>scripts/step4_10.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -273,7 +273,7 @@
<action name="Step5_1"> <action name="Step5_1">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step5_1.sql</script> <script>scripts/step5_1.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -284,7 +284,7 @@
<action name="Step5_2"> <action name="Step5_2">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step5_2.sql</script> <script>scripts/step5_2.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -295,7 +295,7 @@
<action name="Step5_3"> <action name="Step5_3">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step5_3.sql</script> <script>scripts/step5_3.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -306,7 +306,7 @@
<action name="Step5_4"> <action name="Step5_4">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step5_4.sql</script> <script>scripts/step5_4.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -317,7 +317,7 @@
<action name="Step5_5"> <action name="Step5_5">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step5_5.sql</script> <script>scripts/step5_5.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -328,7 +328,7 @@
<action name="Step5_6"> <action name="Step5_6">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step5_6.sql</script> <script>scripts/step5_6.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -339,7 +339,7 @@
<action name="Step5_7"> <action name="Step5_7">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step5_7.sql</script> <script>scripts/step5_7.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -350,7 +350,7 @@
<action name="Step5_8"> <action name="Step5_8">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step5_8.sql</script> <script>scripts/step5_8.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -361,7 +361,7 @@
<action name="Step5_9"> <action name="Step5_9">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step5_9.sql</script> <script>scripts/step5_9.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -372,7 +372,7 @@
<action name="Step5_10"> <action name="Step5_10">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step5_10.sql</script> <script>scripts/step5_10.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -383,7 +383,7 @@
<action name="Step6_1"> <action name="Step6_1">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step6_1.sql</script> <script>scripts/step6_1.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -394,7 +394,7 @@
<action name="Step6_2"> <action name="Step6_2">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step6_2.sql</script> <script>scripts/step6_2.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -405,7 +405,7 @@
<action name="Step6_3"> <action name="Step6_3">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step6_3.sql</script> <script>scripts/step6_3.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -416,7 +416,7 @@
<action name="Step6_4"> <action name="Step6_4">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step6_4.sql</script> <script>scripts/step6_4.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -427,7 +427,7 @@
<action name="Step6_5"> <action name="Step6_5">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step6_5.sql</script> <script>scripts/step6_5.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -438,7 +438,7 @@
<action name="Step6_6"> <action name="Step6_6">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step6_6.sql</script> <script>scripts/step6_6.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -449,7 +449,7 @@
<action name="Step6_7"> <action name="Step6_7">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step6_7.sql</script> <script>scripts/step6_7.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -460,7 +460,7 @@
<action name="Step6_8"> <action name="Step6_8">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step6_8.sql</script> <script>scripts/step6_8.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -471,7 +471,7 @@
<action name="Step6_9"> <action name="Step6_9">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step6_9.sql</script> <script>scripts/step6_9.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -482,7 +482,7 @@
<action name="Step6_10"> <action name="Step6_10">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step6_10.sql</script> <script>scripts/step6_10.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -493,7 +493,7 @@
<action name="Step7_1"> <action name="Step7_1">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step7_1.sql</script> <script>scripts/step7_1.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -504,7 +504,7 @@
<action name="Step7_2"> <action name="Step7_2">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step7_2.sql</script> <script>scripts/step7_2.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -515,7 +515,7 @@
<action name="Step7_3"> <action name="Step7_3">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step7_3.sql</script> <script>scripts/step7_3.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -526,7 +526,7 @@
<action name="Step7_4"> <action name="Step7_4">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step7_4.sql</script> <script>scripts/step7_4.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -537,7 +537,7 @@
<action name="Step7_5"> <action name="Step7_5">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step7_5.sql</script> <script>scripts/step7_5.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -548,7 +548,7 @@
<action name="Step8_1"> <action name="Step8_1">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_1.sql</script> <script>scripts/step8_1.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -559,7 +559,7 @@
<action name="Step8_2"> <action name="Step8_2">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_2.sql</script> <script>scripts/step8_2.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -570,7 +570,7 @@
<action name="Step8_3"> <action name="Step8_3">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_3.sql</script> <script>scripts/step8_3.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -581,7 +581,7 @@
<action name="Step8_4"> <action name="Step8_4">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_4.sql</script> <script>scripts/step8_4.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -592,7 +592,7 @@
<action name="Step8_5"> <action name="Step8_5">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_5.sql</script> <script>scripts/step8_5.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -603,7 +603,7 @@
<action name="Step8_6"> <action name="Step8_6">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_6.sql</script> <script>scripts/step8_6.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -614,7 +614,7 @@
<action name="Step8_7"> <action name="Step8_7">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_7.sql</script> <script>scripts/step8_7.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -625,7 +625,7 @@
<action name="Step8_8"> <action name="Step8_8">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_8.sql</script> <script>scripts/step8_8.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -636,7 +636,7 @@
<action name="Step8_9"> <action name="Step8_9">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_9.sql</script> <script>scripts/step8_9.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -647,7 +647,7 @@
<action name="Step8_10"> <action name="Step8_10">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_10.sql</script> <script>scripts/step8_10.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -658,7 +658,7 @@
<action name="Step8_11"> <action name="Step8_11">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step8_11.sql</script> <script>scripts/step8_11.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -669,7 +669,7 @@
<action name="Step9_1"> <action name="Step9_1">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step9_1.sql</script> <script>scripts/step9_1.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -680,7 +680,7 @@
<action name="Step9_2"> <action name="Step9_2">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step9_2.sql</script> <script>scripts/step9_2.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -691,7 +691,7 @@
<action name="Step9_3"> <action name="Step9_3">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step9_3.sql</script> <script>scripts/step9_3.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -702,7 +702,7 @@
<action name="Step9_4"> <action name="Step9_4">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step9_4.sql</script> <script>scripts/step9_4.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -713,7 +713,7 @@
<action name="Step9_5"> <action name="Step9_5">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step9_5.sql</script> <script>scripts/step9_5.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -724,7 +724,7 @@
<action name="Step9_6"> <action name="Step9_6">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step9_6.sql</script> <script>scripts/step9_6.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -735,7 +735,7 @@
<action name="Step9_7"> <action name="Step9_7">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step9_7.sql</script> <script>scripts/step9_7.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -746,7 +746,7 @@
<action name="Step9_8"> <action name="Step9_8">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step9_8.sql</script> <script>scripts/step9_8.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -757,7 +757,7 @@
<action name="Step9_9"> <action name="Step9_9">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step9_9.sql</script> <script>scripts/step9_9.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -768,7 +768,7 @@
<action name="Step10_1"> <action name="Step10_1">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step10_1.sql</script> <script>scripts/step10_1.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -779,7 +779,7 @@
<action name="Step10_2"> <action name="Step10_2">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step10_2.sql</script> <script>scripts/step10_2.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -790,7 +790,7 @@
<action name="Step10_3"> <action name="Step10_3">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step10_3.sql</script> <script>scripts/step10_3.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -801,7 +801,7 @@
<action name="Step11"> <action name="Step11">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step11.sql</script> <script>scripts/step11.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -813,7 +813,7 @@
<action name="Step12_1"> <action name="Step12_1">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step12_1.sql</script> <script>scripts/step12_1.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -824,7 +824,7 @@
<action name="Step12_2"> <action name="Step12_2">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step12_2.sql</script> <script>scripts/step12_2.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -835,7 +835,7 @@
<action name="Step12_3"> <action name="Step12_3">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step12_3.sql</script> <script>scripts/step12_3.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -846,7 +846,7 @@
<action name="Step12_4"> <action name="Step12_4">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step12_4.sql</script> <script>scripts/step12_4.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -857,7 +857,7 @@
<action name="Step12_5"> <action name="Step12_5">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step12_5.sql</script> <script>scripts/step12_5.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -868,7 +868,7 @@
<action name="Step12_6"> <action name="Step12_6">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step12_6.sql</script> <script>scripts/step12_6.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -879,7 +879,7 @@
<action name="Step12_7"> <action name="Step12_7">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step12_7.sql</script> <script>scripts/step12_7.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -890,7 +890,7 @@
<action name="Step12_8"> <action name="Step12_8">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step12_8.sql</script> <script>scripts/step12_8.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -901,7 +901,7 @@
<action name="Step13"> <action name="Step13">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step13.sql</script> <script>scripts/step13.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>
@ -912,7 +912,7 @@
<action name="Step14"> <action name="Step14">
<hive2 xmlns="uri:oozie:hive2-action:0.1"> <hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url> <jdbc-url>${hiveJdbcUrl}</jdbc-url>
<script>scripts/step14.sql</script> <script>scripts/step14.sql</script>
<param>stats_db_name=${stats_db_name}</param> <param>stats_db_name=${stats_db_name}</param>
<param>openaire_db_name=${openaire_db_name}</param> <param>openaire_db_name=${openaire_db_name}</param>