+ * Above example will produce avro datastore in /report/path
+ * with single {@link ReportEntry}.
+ * Where the {@link ReportEntry#getKey()} will be equal to someProperty and
+ * the {@link ReportEntry#getValue()} will be equal to someValue
+ * (notice the stripped report. prefix from the entry key).
+ *
+ *
+ * @author madryk
+ *
+ */
+public class ReportGenerator implements Process {
+
+ private static final String REPORT_PORT_OUT_NAME = "report";
+
+ private static final String REPORT_PROPERTY_PREFIX = "report.";
+
+
+ //------------------------ LOGIC --------------------------
+
+ @Override
+ public Map getInputPorts() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Map getOutputPorts() {
+ return Collections.singletonMap(REPORT_PORT_OUT_NAME, new AvroPortType(ReportEntry.SCHEMA$));
+ }
+
+ @Override
+ public void run(PortBindings portBindings, Configuration conf, Map parameters) throws Exception {
+
+ Map entriesToReport = collectEntriesToReport(parameters);
+
+ List avroReport = convertToAvroReport(entriesToReport);
+
+
+ FileSystem fs = FileSystem.get(conf);
+
+ Path reportPath = portBindings.getOutput().get(REPORT_PORT_OUT_NAME);
+
+ DataStore.create(avroReport, new FileSystemPath(fs, reportPath));
+
+ }
+
+
+ //------------------------ PRIVATE --------------------------
+
+ private Map collectEntriesToReport(Map parameters) {
+
+ return parameters.entrySet().stream()
+ .filter(property -> property.getKey().startsWith(REPORT_PROPERTY_PREFIX))
+ .map(x -> Pair.of(x.getKey().substring(REPORT_PROPERTY_PREFIX.length()), x.getValue()))
+ .collect(Collectors.toMap(e -> e.getLeft(), e -> e.getRight()));
+
+ }
+
+ private List convertToAvroReport(Map entriesToReport) {
+
+ List avroReport = Lists.newArrayList();
+ entriesToReport.forEach((key, value) -> avroReport.add(ReportEntryFactory.createCounterReportEntry(key, Long.valueOf(value))));
+
+ return avroReport;
+ }
+
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/spark/pipe/SparkPipeExecutor.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/spark/pipe/SparkPipeExecutor.java
new file mode 100644
index 0000000000..33b7f788c5
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/spark/pipe/SparkPipeExecutor.java
@@ -0,0 +1,74 @@
+package eu.dnetlib.dhp.common.spark.pipe;
+
+import java.io.Serializable;
+
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.mapred.AvroKey;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.spark.SparkFiles;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+
+import eu.dnetlib.dhp.common.utils.AvroGsonFactory;
+import scala.Tuple2;
+
+
+/**
+ * Executor of mapreduce scripts using spark pipes.
+ * It imitates hadoop streaming behavior.
+ *
+ * @author madryk
+ *
+ */
+public class SparkPipeExecutor implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+
+ //------------------------ LOGIC --------------------------
+
+ /**
+ * Imitates map part of hadoop streaming job.
+ * It executes provided script for every key in inputRecords rdd.
+ *
+ * It is assumed that provided script will read records from standard input (one line for one record)
+ * and write mapped record into standard output (also one line for one record).
+ * Mapped record can be a key/value pair. In that case script should return key and value
+ * splitted by tab (\t) character in single line.
+ */
+ public JavaPairRDD doMap(JavaPairRDD, NullWritable> inputRecords, String scriptName, String args) {
+
+ JavaRDD mappedRecords = inputRecords.keys().pipe("python " + SparkFiles.get(scriptName) + " " + args);
+
+ JavaPairRDD outputRecords = mappedRecords
+ .mapToPair(line -> {
+ String[] splittedPair = line.split("\t");
+ return new Tuple2(splittedPair[0], (splittedPair.length == 1) ? null : splittedPair[1]);
+ });
+
+ return outputRecords;
+ }
+
+ /**
+ * Imitates reduce part of hadoop streaming job.
+ *
+ * It is assumed that provided script will read records from standard input (one line for one record)
+ * and group records with the same key into single record (reduce).
+ * Method assures that all input records with the same key will be transfered in adjacent lines.
+ * Reduced records should be written by script into standard output (one line for one record).
+ * Reduced records must be json strings of class provided as argument.
+ */
+ public JavaPairRDD, NullWritable> doReduce(JavaPairRDD inputRecords, String scriptName, String args, Class outputClass) {
+
+ JavaRDD reducedRecords = inputRecords.sortByKey()
+ .map(record -> record._1 + ((record._2 == null) ? "" : ("\t" + record._2)))
+ .pipe("python " + SparkFiles.get(scriptName) + " " + args);
+
+ JavaPairRDD, NullWritable> outputRecords = reducedRecords
+ .map(recordString -> AvroGsonFactory.create().fromJson(recordString, outputClass))
+ .mapToPair(record -> new Tuple2, NullWritable>(new AvroKey<>(record), NullWritable.get()));
+
+ return outputRecords;
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/CharSequenceUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/CharSequenceUtils.java
new file mode 100644
index 0000000000..5889cf57a3
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/CharSequenceUtils.java
@@ -0,0 +1,31 @@
+package eu.dnetlib.dhp.common.string;
+
+/**
+ * Operations on {@link CharSequence}
+ *
+ * @author Łukasz Dumiszewski
+*/
+
+public final class CharSequenceUtils {
+
+
+ //------------------------ CONSTRUCTORS --------------------------
+
+ private CharSequenceUtils() {
+ throw new IllegalStateException("may not be initialized");
+ }
+
+
+ //------------------------ LOGIC --------------------------
+
+ /**
+ * Converts the given {@link CharSequence} value to {@link String} by using {@link CharSequence#toString()}.
+ * Returns empty string if value is null.
+ */
+ public static String toStringWithNullToEmpty(CharSequence value) {
+
+ return value == null? "": value.toString();
+
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/DiacriticsRemover.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/DiacriticsRemover.java
new file mode 100644
index 0000000000..b69ed34199
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/DiacriticsRemover.java
@@ -0,0 +1,113 @@
+/*
+ * This file is part of CoAnSys project.
+ * Copyright (c) 2012-2015 ICM-UW
+ *
+ * CoAnSys is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+
+ * CoAnSys is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with CoAnSys. If not, see .
+ */
+
+package eu.dnetlib.dhp.common.string;
+
+import java.text.Normalizer;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Mapping to the basic Latin alphabet (a-z, A-Z). In most cases, a character is
+ * mapped to the closest visual form, rather than functional one, e.g.: "ö" is
+ * mapped to "o" rather than "oe", and "đ" is mapped to "d" rather than "dj" or
+ * "gj". Notable exceptions include: "ĸ" mapped to "q", "ß" mapped to "ss", and
+ * "Þ", "þ" mapped to "Y", "y".
+ *
+ *
Each character is processed as follows:
the character is
+ * compatibility decomposed,
all the combining marks are removed,
+ *
the character is compatibility composed,
additional "manual"
+ * substitutions are applied.
+ *
+ *
All the characters from the "Latin-1 Supplement" and "Latin Extended-A"
+ * Unicode blocks are mapped to the "Basic Latin" block. Characters from other
+ * alphabets are generally left intact, although the decomposable ones may be
+ * affected by the procedure.
+ *
+ * @author Lukasz Bolikowski (bolo@icm.edu.pl)
+ *
+ * @author Łukasz Dumiszewski /just copied from coansys-commons/
+ *
+ */
+public final class DiacriticsRemover {
+
+ private static final Character[] from = {
+ 'Æ', 'Ð', 'Ø', 'Þ', 'ß', 'æ', 'ð', 'ø', 'þ', 'Đ', 'đ', 'Ħ',
+ 'ħ', 'ı', 'ĸ', 'Ł', 'ł', 'Ŋ', 'ŋ', 'Œ', 'œ', 'Ŧ', 'ŧ'};
+ private static final String[] to = {
+ "AE", "D", "O", "Y", "ss", "ae", "d", "o", "y", "D", "d", "H",
+ "h", "i", "q", "L", "l", "N", "n", "OE", "oe", "T", "t"};
+
+ private static Map lookup = buildLookup();
+
+
+ //------------------------ CONSTRUCTORS -------------------
+
+
+ private DiacriticsRemover() {}
+
+
+ //------------------------ LOGIC --------------------------
+
+
+ /**
+ * Removes diacritics from a text.
+ *
+ * @param text Text to process.
+ * @return Text without diacritics.
+ */
+ public static String removeDiacritics(String text) {
+ if (text == null) {
+ return null;
+ }
+
+ String tmp = Normalizer.normalize(text, Normalizer.Form.NFKD);
+
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < tmp.length(); i++) {
+ Character ch = tmp.charAt(i);
+ if (Character.getType(ch) == Character.NON_SPACING_MARK) {
+ continue;
+ }
+
+ if (lookup.containsKey(ch)) {
+ builder.append(lookup.get(ch));
+ } else {
+ builder.append(ch);
+ }
+ }
+
+ return builder.toString();
+ }
+
+
+ //------------------------ PRIVATE --------------------------
+
+ private static Map buildLookup() {
+ if (from.length != to.length) {
+ throw new IllegalStateException();
+ }
+
+ Map _lookup = new HashMap();
+ for (int i = 0; i < from.length; i++) {
+ _lookup.put(from[i], to[i]);
+ }
+
+ return _lookup;
+ }
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/LenientComparisonStringNormalizer.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/LenientComparisonStringNormalizer.java
new file mode 100644
index 0000000000..bae64ae38b
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/LenientComparisonStringNormalizer.java
@@ -0,0 +1,130 @@
+/*
+ * This file is part of CoAnSys project.
+ * Copyright (c) 2012-2015 ICM-UW
+ *
+ * CoAnSys is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+
+ * CoAnSys is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with CoAnSys. If not, see .
+ */
+package eu.dnetlib.dhp.common.string;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * An implementation of {@link StringNormalizer} that normalizes strings for non-strict comparisons
+ * in which one does not care about characters other than letters and digits or about differently written diacritics.
+ *
+ * @author Łukasz Dumiszewski
+ *
+ */
+public final class LenientComparisonStringNormalizer implements StringNormalizer, Serializable {
+
+
+ private static final long serialVersionUID = 1L;
+
+
+ private List whitelistCharacters;
+
+
+ //------------------------ CONSTRUCTORS --------------------------
+
+ public LenientComparisonStringNormalizer() {
+ this(ImmutableList.of());
+ }
+
+ /**
+ * @param whitelistCharacters - non alphanumeric characters that will not be removed
+ * during normalization
+ */
+ public LenientComparisonStringNormalizer(List whitelistCharacters) {
+ this.whitelistCharacters = whitelistCharacters;
+ }
+
+
+ //------------------------ LOGIC --------------------------
+
+
+
+ /**
+ * Normalizes the given value.
+ * The normalized strings are better suited for non-strict comparisons, in which one does NOT care about characters that are
+ * neither letters nor digits; about accidental spaces or different diacritics etc.
+ * This method:
+ *
+ *
Replaces all characters that are not letters or digits with spaces (except those on whitelist characters list)
+ *
Replaces white spaces with spaces
+ *
Trims
+ *
Compacts multi-space gaps to one-space gaps
+ *
Removes diacritics
+ *
Changes characters to lower case
+ *
+ * Returns "" if the passed value is null or blank
+ *
+ * @param value the string to normalize
+ * @see DiacriticsRemover#removeDiacritics(String, boolean)
+ *
+ *
+ */
+ public String normalize(String value) {
+
+ if (StringUtils.isBlank(value)) {
+
+ return "";
+
+ }
+
+
+ String result = value;
+
+ result = DiacriticsRemover.removeDiacritics(result);
+
+ result = removeNonLetterDigitCharacters(result);
+
+ result = result.toLowerCase();
+
+ result = result.trim().replaceAll(" +", " ");
+
+ return result;
+ }
+
+
+
+
+ //------------------------ PRIVATE --------------------------
+
+
+ private String removeNonLetterDigitCharacters(final String value) {
+
+ StringBuilder sb = new StringBuilder();
+
+ for (int i = 0; i < value.length(); ++i) {
+
+ char c = value.charAt(i);
+
+ if (Character.isLetterOrDigit(c) || whitelistCharacters.contains(c)) {
+ sb.append(c);
+ } else {
+ sb.append(' ');
+ }
+ }
+
+ return sb.toString();
+ }
+
+
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/StringNormalizer.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/StringNormalizer.java
new file mode 100644
index 0000000000..6e28422bc0
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/StringNormalizer.java
@@ -0,0 +1,16 @@
+package eu.dnetlib.dhp.common.string;
+
+/**
+ * String normalizer.
+ *
+ * @author Łukasz Dumiszewski
+ *
+ */
+public interface StringNormalizer {
+
+ /**
+ * Normalizes the given string value.
+ */
+ String normalize(String value);
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroGsonFactory.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroGsonFactory.java
new file mode 100644
index 0000000000..7fcc0506a0
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroGsonFactory.java
@@ -0,0 +1,45 @@
+package eu.dnetlib.dhp.common.utils;
+
+import java.lang.reflect.Type;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
+
+/**
+ * Factory for gson object that supports serializing avro generated classes
+ *
+ * @author madryk
+ *
+ */
+public final class AvroGsonFactory {
+
+ //------------------------ CONSTRUCTORS -------------------
+
+
+ private AvroGsonFactory() {}
+
+
+ //------------------------ LOGIC --------------------------
+
+ public static Gson create() {
+ GsonBuilder builder = new GsonBuilder();
+
+ builder.registerTypeAdapter(CharSequence.class, new CharSequenceDeserializer());
+
+ return builder.create();
+ }
+
+ public static class CharSequenceDeserializer implements JsonDeserializer {
+
+ @Override
+ public CharSequence deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
+ throws JsonParseException {
+ return json.getAsString();
+ }
+
+ }
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroUtils.java
new file mode 100644
index 0000000000..44dd218b5e
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroUtils.java
@@ -0,0 +1,77 @@
+package eu.dnetlib.dhp.common.utils;
+
+import java.lang.reflect.Field;
+
+import org.apache.avro.Schema;
+
+/**
+ *
+ * @author Mateusz Kobos
+ *
+ */
+public final class AvroUtils {
+
+ public final static String primitiveTypePrefix = "org.apache.avro.Schema.Type.";
+
+
+ //------------------------ CONSTRUCTORS -------------------
+
+
+ private AvroUtils() {}
+
+
+ //------------------------ LOGIC --------------------------
+
+
+ /**
+ * For a given name of a class generated from Avro schema return
+ * a JSON schema.
+ *
+ * Apart from a name of a class you can also give a name of one of enums
+ * defined in {@link org.apache.avro.Schema.Type}; in such case an
+ * appropriate primitive type will be returned.
+ *
+ * @param typeName fully qualified name of a class generated from Avro schema,
+ * e.g. {@code eu.dnetlib.dhp.common.avro.Person},
+ * or a fully qualified name of enum defined by
+ * {@link org.apache.avro.Schema.Type},
+ * e.g. {@link org.apache.avro.Schema.Type.STRING}.
+ * @return JSON string
+ */
+ public static Schema toSchema(String typeName) {
+ Schema schema = null;
+ if(typeName.startsWith(primitiveTypePrefix)){
+ String shortName = typeName.substring(
+ primitiveTypePrefix.length(), typeName.length());
+ schema = getPrimitiveTypeSchema(shortName);
+ } else {
+ schema = getAvroClassSchema(typeName);
+ }
+ return schema;
+ }
+
+ private static Schema getPrimitiveTypeSchema(String shortName){
+ Schema.Type type = Schema.Type.valueOf(shortName);
+ return Schema.create(type);
+ }
+
+ private static Schema getAvroClassSchema(String className){
+ try {
+ Class avroClass = Class.forName(className);
+ Field f = avroClass.getDeclaredField("SCHEMA$");
+ return (Schema) f.get(null);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(
+ "Class \""+className+"\" does not exist", e);
+ } catch (SecurityException e) {
+ throw new RuntimeException(e);
+ } catch (NoSuchFieldException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalArgumentException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/ByteArrayUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/ByteArrayUtils.java
new file mode 100644
index 0000000000..152271ab7c
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/ByteArrayUtils.java
@@ -0,0 +1,45 @@
+package eu.dnetlib.dhp.common.utils;
+
+/**
+ * Byte array utility class.
+ * @author mhorst
+ *
+ */
+public final class ByteArrayUtils {
+
+ //------------------------ CONSTRUCTORS -------------------
+
+ private ByteArrayUtils() {}
+
+ //------------------------ LOGIC --------------------------
+
+ /**
+ * Does this byte array begin with match array content?
+ * @param source Byte array to examine
+ * @param match Byte array to locate in source
+ * @return true If the starting bytes are equal
+ */
+ public static boolean startsWith(byte[] source, byte[] match) {
+ return startsWith(source, 0, match);
+ }
+
+ /**
+ * Does this byte array begin with match array content?
+ * @param source Byte array to examine
+ * @param offset An offset into the source array
+ * @param match Byte array to locate in source
+ * @return true If the starting bytes are equal
+ */
+ public static boolean startsWith(byte[] source, int offset, byte[] match) {
+ if (match.length > (source.length - offset)) {
+ return false;
+ }
+ for (int i = 0; i < match.length; i++) {
+ if (source[offset + i] != match[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+}
diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/EmptyDatastoreVerifierProcess.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/EmptyDatastoreVerifierProcess.java
new file mode 100644
index 0000000000..1e6e041498
--- /dev/null
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/EmptyDatastoreVerifierProcess.java
@@ -0,0 +1,89 @@
+package eu.dnetlib.dhp.common.utils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.security.InvalidParameterException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import eu.dnetlib.dhp.common.java.PortBindings;
+import eu.dnetlib.dhp.common.java.Ports;
+import eu.dnetlib.dhp.common.java.Process;
+import eu.dnetlib.dhp.common.java.io.CloseableIterator;
+import eu.dnetlib.dhp.common.java.io.DataStore;
+import eu.dnetlib.dhp.common.java.io.FileSystemPath;
+import eu.dnetlib.dhp.common.java.porttype.AnyPortType;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.OOZIE_ACTION_OUTPUT_FILENAME;
+
+/**
+ * Simple process verifying whether given datastore is empty.
+ * @author mhorst
+ *
+ */
+public class EmptyDatastoreVerifierProcess implements Process {
+
+ public static final String INPUT_PORT_NAME = "input";
+
+ public static final String DEFAULT_ENCODING = "UTF-8";
+
+ public static final String OUTPUT_PROPERTY_IS_EMPTY = "isEmpty";
+
+ /**
+ * Ports handled by this module.
+ */
+ private final Ports ports;
+
+
+ // ------------------------ CONSTRUCTORS --------------------------
+
+ public EmptyDatastoreVerifierProcess() {
+// preparing ports
+ Map input = new HashMap();
+ input.put(INPUT_PORT_NAME, new AnyPortType());
+ Map output = Collections.emptyMap();
+ ports = new Ports(input, output);
+ }
+
+ @Override
+ public Map getInputPorts() {
+ return ports.getInput();
+ }
+
+ @Override
+ public Map getOutputPorts() {
+ return ports.getOutput();
+ }
+
+ @Override
+ public void run(PortBindings portBindings, Configuration conf, Map parameters) throws Exception {
+ if (!portBindings.getInput().containsKey(INPUT_PORT_NAME)) {
+ throw new InvalidParameterException("missing input port!");
+ }
+
+ try (CloseableIterator closeableIt = getIterator(conf, portBindings.getInput().get(INPUT_PORT_NAME))) {
+ File file = new File(System.getProperty(OOZIE_ACTION_OUTPUT_FILENAME));
+ Properties props = new Properties();
+ props.setProperty(OUTPUT_PROPERTY_IS_EMPTY, Boolean.toString(!closeableIt.hasNext()));
+ try (OutputStream os = new FileOutputStream(file)) {
+ props.store(os, "");
+ }
+ }
+ }
+
+ /**
+ * Returns iterator over datastore.
+ */
+ protected CloseableIterator getIterator(Configuration conf, Path path) throws IOException {
+ return DataStore.getReader(new FileSystemPath(FileSystem.get(conf), path));
+ }
+
+}
diff --git a/dhp-schemas/README.md b/dhp-schemas/README.md
new file mode 100644
index 0000000000..473ad4cf19
--- /dev/null
+++ b/dhp-schemas/README.md
@@ -0,0 +1,3 @@
+Description of the project
+--------------------------
+This project defines **serialization schemas** of Avro data store files that are used to pass data between workflow nodes in the system.
diff --git a/dhp-schemas/pom.xml b/dhp-schemas/pom.xml
new file mode 100644
index 0000000000..2c6e18f271
--- /dev/null
+++ b/dhp-schemas/pom.xml
@@ -0,0 +1,62 @@
+
+
+ 4.0.0
+
+
+ eu.dnetlib.dhp
+ dhp
+ 1.0.0-SNAPSHOT
+
+
+ dhp-schemas
+ jar
+
+
+
+ org.apache.avro
+ avro
+
+
+
+
+
+
+
+ org.apache.avro
+ avro-maven-plugin
+
+
+ generate-sources
+
+ schema
+ idl-protocol
+
+
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+
+
+ add-source
+ generate-sources
+
+ add-source
+
+
+
+ ${project.build.directory}/generated-sources/avro/
+
+
+
+
+
+
+
+
+
diff --git a/dhp-schemas/src/main/avro/eu/dnetlib/dhp/audit/Fault.avdl b/dhp-schemas/src/main/avro/eu/dnetlib/dhp/audit/Fault.avdl
new file mode 100644
index 0000000000..3bce821a4b
--- /dev/null
+++ b/dhp-schemas/src/main/avro/eu/dnetlib/dhp/audit/Fault.avdl
@@ -0,0 +1,29 @@
+@namespace("eu.dnetlib.dhp.audit.schemas")
+protocol DHP {
+
+ record Cause {
+// generic cause code, root exception class name when derived from exception
+ string code;
+// cause message
+ union { null , string } message = null;
+ }
+
+ record Fault {
+// input object identifier
+ string inputObjectId;
+// fault creation timestamp
+ long timestamp;
+// generic fault code, root exception class name when derived from exception
+ string code;
+// fault message
+ union { null , string } message = null;
+// stack trace
+ union { null , string } stackTrace = null;
+// fault causes, array is indexed with cause depth
+ union { null , array } causes = null;
+// Other supplementary data related to specific type of fault.
+// See parameters description in oozie workflow.xml documentation of modules
+// that use this structure for information what exactly can be stored as supplementary data.
+ union { null , map } supplementaryData = null;
+ }
+}
diff --git a/dhp-schemas/src/main/avro/eu/dnetlib/dhp/common/ReportEntry.avdl b/dhp-schemas/src/main/avro/eu/dnetlib/dhp/common/ReportEntry.avdl
new file mode 100644
index 0000000000..99406b4f06
--- /dev/null
+++ b/dhp-schemas/src/main/avro/eu/dnetlib/dhp/common/ReportEntry.avdl
@@ -0,0 +1,16 @@
+@namespace("eu.dnetlib.dhp.common.schemas")
+protocol DHP{
+
+ enum ReportEntryType {
+ COUNTER, DURATION
+ }
+
+
+ record ReportEntry {
+
+ string key;
+ ReportEntryType type;
+ string value;
+
+ }
+}
diff --git a/dhp-schemas/src/main/avro/eu/dnetlib/dhp/importer/NativeRecord.avdl b/dhp-schemas/src/main/avro/eu/dnetlib/dhp/importer/NativeRecord.avdl
new file mode 100644
index 0000000000..9ad5435fac
--- /dev/null
+++ b/dhp-schemas/src/main/avro/eu/dnetlib/dhp/importer/NativeRecord.avdl
@@ -0,0 +1,21 @@
+@namespace("eu.dnetlib.dhp.importer.schemas")
+protocol DHP {
+
+ enum RecordFormat {
+ XML, JSON
+ }
+
+ record ImportedRecord {
+
+ // record identifier
+ string id;
+
+ RecordFormat format;
+
+ // format name (OAF, OAI_DC, Datacite, etc) for which there is a parser implementation
+ string formatName;
+
+ // record body
+ string body;
+ }
+}
diff --git a/dhp-wf/dhp-wf-import/pom.xml b/dhp-wf/dhp-wf-import/pom.xml
new file mode 100644
index 0000000000..6bf4ba8250
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/pom.xml
@@ -0,0 +1,105 @@
+
+
+
+ eu.dnetlib.dhp
+ dhp-wf
+ 1.0.0-SNAPSHOT
+
+ 4.0.0
+
+ dhp-wf-import
+
+
+
+
+ ${project.groupId}
+ dhp-common
+ ${project.version}
+
+
+
+ ${project.groupId}
+ dhp-common
+ ${project.version}
+ test-jar
+ test
+
+
+
+ ${project.groupId}
+ dhp-schemas
+ ${project.version}
+
+
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+
+ com.googlecode.json-simple
+ json-simple
+
+
+ commons-cli
+ commons-cli
+
+
+ eu.dnetlib
+ dnet-objectstore-rmi
+
+
+ eu.dnetlib
+ cnr-rmi-api
+
+
+ eu.dnetlib
+ cnr-resultset-client
+
+
+ eu.dnetlib
+ dnet-openaireplus-mapping-utils
+
+
+
+ org.springframework
+ spring-context
+
+
+ org.apache.cxf
+ cxf-rt-frontend-jaxws
+
+
+ com.google.code.gson
+ gson
+
+
+
+ org.apache.spark
+ spark-core_2.10
+
+
+ org.apache.spark
+ spark-sql_2.10
+
+
+ com.databricks
+ spark-avro_2.10
+
+
+ org.mongodb.spark
+ mongo-spark-connector_2.10
+
+
+
+
+
+
+
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.java
new file mode 100644
index 0000000000..214d6691dd
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.java
@@ -0,0 +1,29 @@
+package eu.dnetlib.dhp.wf.importer;
+
+import java.io.IOException;
+
+import org.apache.avro.file.DataFileWriter;
+
+/**
+ * {@link DataFileWriter} based record receiver.
+ * @author mhorst
+ *
+ */
+public class DataFileRecordReceiver implements RecordReceiver {
+
+ private final DataFileWriter writer;
+
+ /**
+ * Default constructor.
+ * @param writer
+ */
+ public DataFileRecordReceiver(DataFileWriter writer) {
+ this.writer = writer;
+ }
+
+ @Override
+ public void receive(T object) throws IOException {
+ this.writer.append(object);
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.java
new file mode 100644
index 0000000000..955f18065c
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.java
@@ -0,0 +1,50 @@
+package eu.dnetlib.dhp.wf.importer;
+
+import java.io.IOException;
+
+import org.apache.avro.file.DataFileWriter;
+
+/**
+ * {@link DataFileWriter} based record receiver with counter of
+ * received records.
+ *
+ * @author madryk
+ */
+public class DataFileRecordReceiverWithCounter extends DataFileRecordReceiver {
+
+ private long receivedCount = 0L;
+
+
+ //------------------------ CONSTRUCTORS --------------------------
+
+ /**
+ * Default constructor
+ *
+ * @param writer - writer of the received records
+ */
+ public DataFileRecordReceiverWithCounter(DataFileWriter writer) {
+ super(writer);
+ }
+
+
+ //------------------------ GETTERS --------------------------
+
+ /**
+ * Returns number of received records
+ */
+ public long getReceivedCount() {
+ return receivedCount;
+ }
+
+
+ //------------------------ LOGIC --------------------------
+
+ /**
+ * Receives passed record and increments the counter.
+ */
+ @Override
+ public void receive(T record) throws IOException {
+ super.receive(record);
+ ++receivedCount;
+ }
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.java
new file mode 100644
index 0000000000..40f673ee08
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.java
@@ -0,0 +1,52 @@
+package eu.dnetlib.dhp.wf.importer;
+
+/**
+ * Import realated workflow parameters.
+ * @author mhorst
+ *
+ */
+public final class ImportWorkflowRuntimeParameters {
+
+ // parameter names
+
+ public static final String IMPORT_INFERENCE_PROVENANCE_BLACKLIST = "import.inference.provenance.blacklist";
+ public static final String IMPORT_SKIP_DELETED_BY_INFERENCE = "import.skip.deleted.by.inference";
+ public static final String IMPORT_TRUST_LEVEL_THRESHOLD = "import.trust.level.threshold";
+ public static final String IMPORT_APPROVED_DATASOURCES_CSV = "import.approved.datasources.csv";
+ public static final String IMPORT_APPROVED_COLUMNFAMILIES_CSV = "import.approved.columnfamilies.csv";
+ public static final String IMPORT_MERGE_BODY_WITH_UPDATES = "import.merge.body.with.updates";
+ public static final String IMPORT_CONTENT_APPROVED_OBJECSTORES_CSV = "import.content.approved.objectstores.csv";
+ public static final String IMPORT_CONTENT_BLACKLISTED_OBJECSTORES_CSV = "import.content.blacklisted.objectstores.csv";
+
+ public static final String IMPORT_CONTENT_OBJECT_STORE_LOC = "import.content.object.store.location";
+ public static final String IMPORT_CONTENT_OBJECT_STORE_IDS_CSV = "import.content.object.store.ids.csv";
+ public static final String IMPORT_CONTENT_MAX_FILE_SIZE_MB = "import.content.max.file.size.mb";
+ public static final String IMPORT_CONTENT_CONNECTION_TIMEOUT = "import.content.connection.timeout";
+ public static final String IMPORT_CONTENT_READ_TIMEOUT = "import.content.read.timeout";
+
+ public static final String IMPORT_MDSTORE_IDS_CSV = "import.mdstore.ids.csv";
+ public static final String IMPORT_MDSTORE_SERVICE_LOCATION = "import.mdstore.service.location";
+ public static final String IMPORT_MDSTORE_RECORD_MAXLENGTH = "import.mdstore.record.maxlength";
+
+ public static final String IMPORT_ISLOOKUP_SERVICE_LOCATION = "import.islookup.service.location";
+ public static final String IMPORT_VOCABULARY_CODE = "import.vocabulary.code";
+ public static final String IMPORT_VOCABULARY_OUTPUT_FILENAME = "import.vocabulary.output.filename";
+
+ public static final String IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT = "import.resultset.client.read.timeout";
+ public static final String IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT = "import.resultset.client.connection.timeout";
+ public static final String IMPORT_RESULT_SET_PAGESIZE = "import.resultset.pagesize";
+
+
+ public static final String HBASE_ENCODING = "hbase.table.encoding";
+
+ public static final String IMPORT_FACADE_FACTORY_CLASS = "import.facade.factory.classname";
+
+ // default values
+
+ public static final String RESULTSET_READ_TIMEOUT_DEFAULT_VALUE = "60000";
+ public static final String RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE = "60000";
+ public static final String RESULTSET_PAGESIZE_DEFAULT_VALUE = "100";
+
+ private ImportWorkflowRuntimeParameters() {}
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/RecordReceiver.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/RecordReceiver.java
new file mode 100644
index 0000000000..c0a5e89507
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/RecordReceiver.java
@@ -0,0 +1,14 @@
+package eu.dnetlib.dhp.wf.importer;
+
+import java.io.IOException;
+
+/**
+ * Record receiver interface.
+ * @author mhorst
+ *
+ * @param
+ */
+public interface RecordReceiver {
+
+ void receive(T object) throws IOException;
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.java
new file mode 100644
index 0000000000..0a3cd6fb4c
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.java
@@ -0,0 +1,104 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import java.util.Map;
+
+import javax.xml.ws.BindingProvider;
+import javax.xml.ws.wsaddressing.W3CEndpointReferenceBuilder;
+
+import org.apache.log4j.Logger;
+
+import eu.dnetlib.enabling.tools.JaxwsServiceResolverImpl;
+
+/**
+ * Abstract class utilized by all WebService facades.
+ * @author mhorst
+ *
+ */
+public abstract class AbstractResultSetAwareWebServiceFacade {
+
+ private final Logger log = Logger.getLogger(this.getClass());
+
+ /**
+ * Web service.
+ */
+ private final T service;
+
+ /**
+ * ResultSet read timeout.
+ */
+ private final long resultSetReadTimeout;
+
+ /**
+ * ResultSet connection timeout.
+ */
+ private final long resultSetConnectionTimeout;
+
+ /**
+ * ResultSet page size.
+ */
+ private final int resultSetPageSize;
+
+
+ //------------------------ CONSTRUCTORS -------------------
+
+ /**
+ * Instantiates underlying service.
+ * @param clazz webservice class
+ * @param serviceLocation webservice location
+ * @param serviceReadTimeout service read timeout
+ * @param serviceConnectionTimeout service connection timeout
+ * @param resultSetReadTimeout resultset read timeout
+ * @param resultSetConnectionTimeout resultset connection timeout
+ * @param resultSetPageSize resultset page size
+ */
+ protected AbstractResultSetAwareWebServiceFacade(Class clazz, String serviceLocation,
+ long serviceReadTimeout, long serviceConnectionTimeout,
+ long resultSetReadTimeout, long resultSetConnectionTimeout, int resultSetPageSize) {
+ W3CEndpointReferenceBuilder eprBuilder = new W3CEndpointReferenceBuilder();
+ eprBuilder.address(serviceLocation);
+ eprBuilder.build();
+ this.service = new JaxwsServiceResolverImpl().getService(clazz, eprBuilder.build());
+ if (this.service instanceof BindingProvider) {
+ log.info(String.format("setting timeouts for %s: read timeout (%s) and connect timeout (%s)",
+ BindingProvider.class, serviceReadTimeout, serviceConnectionTimeout));
+ final Map requestContext = ((BindingProvider) service).getRequestContext();
+
+ // can't be sure about which will be used. Set them all.
+ requestContext.put("com.sun.xml.internal.ws.request.timeout", serviceReadTimeout);
+ requestContext.put("com.sun.xml.internal.ws.connect.timeout", serviceConnectionTimeout);
+
+ requestContext.put("com.sun.xml.ws.request.timeout", serviceReadTimeout);
+ requestContext.put("com.sun.xml.ws.connect.timeout", serviceConnectionTimeout);
+
+ requestContext.put("javax.xml.ws.client.receiveTimeout", serviceReadTimeout);
+ requestContext.put("javax.xml.ws.client.connectionTimeout", serviceConnectionTimeout);
+ }
+
+ this.resultSetReadTimeout = resultSetReadTimeout;
+ this.resultSetConnectionTimeout = resultSetConnectionTimeout;
+ this.resultSetPageSize = resultSetPageSize;
+ }
+
+
+ //------------------------ GETTERS -------------------------
+
+ public T getService() {
+ return service;
+ }
+
+
+ public long getResultSetReadTimeout() {
+ return resultSetReadTimeout;
+ }
+
+
+ public long getResultSetConnectionTimeout() {
+ return resultSetConnectionTimeout;
+ }
+
+
+ public int getResultSetPageSize() {
+ return resultSetPageSize;
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.java
new file mode 100644
index 0000000000..c156ae1cc8
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.java
@@ -0,0 +1,17 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+/**
+ * ISLookup service facade.
+ *
+ * @author mhorst
+ *
+ */
+public interface ISLookupFacade {
+
+ /**
+ * Provides all profiles matching given query
+ * @param xPathQuery XPath query
+ */
+ Iterable searchProfile(String xPathQuery) throws ServiceFacadeException;
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.java
new file mode 100644
index 0000000000..f50b02b980
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.java
@@ -0,0 +1,17 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+/**
+ * MDStore service facade.
+ *
+ * @author mhorst
+ *
+ */
+public interface MDStoreFacade {
+
+ /**
+ * Delivers all records for given MDStore identifier
+ * @param mdStoreId MDStore identifier
+ */
+ Iterable deliverMDRecords(String mdStoreId) throws ServiceFacadeException;
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.java
new file mode 100644
index 0000000000..0e1aa19ef9
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.java
@@ -0,0 +1,19 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+/**
+ * ObjectStore service facade.
+ *
+ * @author mhorst
+ *
+ */
+public interface ObjectStoreFacade {
+
+ /**
+ * Returns metadata records from given objectstore created in specified time range.
+ * @param objectStoreId object store identifier
+ * @param from from time in millis
+ * @param until until time in millis
+ */
+ Iterable deliverObjects(String objectStoreId, long from, long until) throws ServiceFacadeException;
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.java
new file mode 100644
index 0000000000..9776306fa2
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.java
@@ -0,0 +1,27 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+/**
+ * Service facade generic exception.
+ *
+ * @author mhorst
+ *
+ */
+public class ServiceFacadeException extends Exception {
+
+ private static final long serialVersionUID = 0L;
+
+ //------------------------ CONSTRUCTORS -------------------
+
+ public ServiceFacadeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ServiceFacadeException(String message) {
+ super(message);
+ }
+
+ public ServiceFacadeException(Throwable cause) {
+ super(cause);
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.java
new file mode 100644
index 0000000000..94b9307c47
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.java
@@ -0,0 +1,20 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import java.util.Map;
+
+/**
+ * Generic service facade factory. All implementations must be instantiable with no-argument construtor.
+ *
+ * @author mhorst
+ *
+ */
+public interface ServiceFacadeFactory {
+
+ /**
+ * Creates service of given type configured with parameters.
+ *
+ * @param parameters service configuration
+ *
+ */
+ T instantiate(Map parameters);
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.java
new file mode 100644
index 0000000000..53a76d761d
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.java
@@ -0,0 +1,80 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_FACADE_FACTORY_CLASS;
+
+import java.lang.reflect.Constructor;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.ImmutableMap;
+
+import eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters;
+
+/**
+ * Service facade utility methods.
+ * @author mhorst
+ *
+ */
+public final class ServiceFacadeUtils {
+
+ //------------------------ CONSTRUCTORS -------------------
+
+ private ServiceFacadeUtils() {}
+
+ //------------------------ LOGIC --------------------------
+
+ /**
+ * Instantiates service based on provided parameters.
+ *
+ * Service factory class name is mandatory and has to be provided as {@value ImportWorkflowRuntimeParameters#IMPORT_FACADE_FACTORY_CLASS} parameter.
+ * Other parameters will be used by factory itself. Factory must be instantiable with no-argument construtor.
+ *
+ * @param parameters set of parameters required for service instantiation
+ *
+ */
+ public static T instantiate(Map parameters) throws ServiceFacadeException {
+ String serviceFactoryClassName = parameters.get(IMPORT_FACADE_FACTORY_CLASS);
+ if (StringUtils.isBlank(serviceFactoryClassName)) {
+ throw new ServiceFacadeException("unknown service facade factory, no " + IMPORT_FACADE_FACTORY_CLASS + " parameter provided!");
+ }
+ try {
+ Class clazz = Class.forName(serviceFactoryClassName);
+ Constructor constructor = clazz.getConstructor();
+ @SuppressWarnings("unchecked")
+ ServiceFacadeFactory serviceFactory = (ServiceFacadeFactory) constructor.newInstance();
+ return serviceFactory.instantiate(parameters);
+ } catch (Exception e) {
+ throw new ServiceFacadeException("exception occurred while instantiating service by facade factory: " + IMPORT_FACADE_FACTORY_CLASS, e);
+ }
+
+ }
+
+ /**
+ * Instantiates service based on provided configuration.
+ *
+ * Service factory class name is mandatory and has to be provided as {@value ImportWorkflowRuntimeParameters#IMPORT_FACADE_FACTORY_CLASS} configuration entry.
+ * Other parameters will be used by factory itself. Factory must be instantiable with no-argument construtor.
+ *
+ * @param config set of configuration entries required for service instantiation
+ */
+ public static T instantiate(Configuration config) throws ServiceFacadeException {
+ return instantiate(buildParameters(config));
+ }
+
+
+ // ------------------------ PRIVATE --------------------------
+
+ /**
+ * Converts configuration entries into plain map.
+ */
+ private static Map buildParameters(Configuration config) {
+ ImmutableMap.Builder builder = ImmutableMap.builder();
+ for (Map.Entry entry : config) {
+ builder.put(entry);
+ }
+ return builder.build();
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.java
new file mode 100644
index 0000000000..7c787f2f8c
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.java
@@ -0,0 +1,55 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import java.util.Collections;
+
+import org.apache.log4j.Logger;
+
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+
+/**
+ * WebService based database facade.
+ *
+ * @author mhorst
+ *
+ */
+public class WebServiceISLookupFacade extends AbstractResultSetAwareWebServiceFacade implements ISLookupFacade {
+
+ private static final Logger log = Logger.getLogger(WebServiceISLookupFacade.class);
+
+
+ //------------------------ CONSTRUCTORS -------------------
+
+ /**
+ * @param serviceLocation database service location
+ * @param serviceReadTimeout service read timeout
+ * @param serviceConnectionTimeout service connection timeout
+ * @param resultSetReadTimeout result set providing database results read timeout
+ * @param resultSetConnectionTimeout result set connection timeout
+ * @param resultSetPageSize result set data chunk size
+ */
+ public WebServiceISLookupFacade(String serviceLocation,
+ long serviceReadTimeout, long serviceConnectionTimeout,
+ long resultSetReadTimeout, long resultSetConnectionTimeout, int resultSetPageSize) {
+ super(ISLookUpService.class, serviceLocation,
+ serviceReadTimeout, serviceConnectionTimeout,
+ resultSetReadTimeout, resultSetConnectionTimeout, resultSetPageSize);
+ }
+
+ //------------------------ LOGIC --------------------------
+
+ @Override
+ public Iterable searchProfile(String xPathQuery) throws ServiceFacadeException {
+ try {
+ return getService().quickSearchProfile(xPathQuery);
+ } catch (ISLookUpDocumentNotFoundException e) {
+ log.error("unable to find profile for query: " + xPathQuery, e);
+ return Collections.emptyList();
+ } catch (ISLookUpException e) {
+ throw new ServiceFacadeException("searching profiles in ISLookup failed with query '" + xPathQuery + "'", e);
+ }
+
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.java
new file mode 100644
index 0000000000..6557ead948
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.java
@@ -0,0 +1,45 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_ISLOOKUP_SERVICE_LOCATION;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_PAGESIZE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_PAGESIZE_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_READ_TIMEOUT_DEFAULT_VALUE;
+
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+import eu.dnetlib.dhp.common.WorkflowRuntimeParameters;
+
+/**
+ * WebService Database service facade factory.
+ *
+ * @author mhorst
+ *
+ */
+public class WebServiceISLookupFacadeFactory implements ServiceFacadeFactory {
+
+
+ //------------------------ LOGIC --------------------------
+
+ @Override
+ public ISLookupFacade instantiate(Map parameters) {
+ Preconditions.checkArgument(parameters.containsKey(IMPORT_ISLOOKUP_SERVICE_LOCATION),
+ "unknown ISLookup service location: no parameter provided: '%s'", IMPORT_ISLOOKUP_SERVICE_LOCATION);
+
+ return new WebServiceISLookupFacade(parameters.get(IMPORT_ISLOOKUP_SERVICE_LOCATION),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_READ_TIMEOUT, DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT, DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT, RESULTSET_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT, RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Integer.parseInt(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_PAGESIZE, RESULTSET_PAGESIZE_DEFAULT_VALUE, parameters)));
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.java
new file mode 100644
index 0000000000..d37d020ed9
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.java
@@ -0,0 +1,52 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import eu.dnetlib.data.mdstore.MDStoreService;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.enabling.tools.JaxwsServiceResolverImpl;
+
+/**
+ * WebService based MDStore facade.
+ *
+ * @author mhorst
+ *
+ */
+public class WebServiceMDStoreFacade extends AbstractResultSetAwareWebServiceFacade implements MDStoreFacade {
+
+
+ //------------------------ CONSTRUCTORS -------------------
+
+ /**
+ * @param serviceLocation MDStore webservice location
+ * @param serviceReadTimeout service read timeout
+ * @param serviceConnectionTimeout service connection timeout
+ * @param resultSetReadTimeout resultset read timeout
+ * @param resultSetConnectionTimeout result set connection timeout
+ * @param resultSetPageSize resultset page size
+ */
+ public WebServiceMDStoreFacade(String serviceLocation,
+ long serviceReadTimeout, long serviceConnectionTimeout,
+ long resultSetReadTimeout, long resultSetConnectionTimeout, int resultSetPageSize) {
+ super(MDStoreService.class, serviceLocation,
+ serviceReadTimeout, serviceConnectionTimeout,
+ resultSetReadTimeout, resultSetConnectionTimeout, resultSetPageSize);
+ }
+
+ //------------------------ LOGIC --------------------------
+
+ @Override
+ public Iterable deliverMDRecords(String mdStoreId) throws ServiceFacadeException {
+ try {
+ W3CEndpointReference eprResult = getService().deliverMDRecords(mdStoreId, null, null, null);
+ ResultSetClientFactory rsFactory = new ResultSetClientFactory(
+ getResultSetPageSize(), getResultSetReadTimeout(), getResultSetConnectionTimeout());
+ rsFactory.setServiceResolver(new JaxwsServiceResolverImpl());
+ return rsFactory.getClient(eprResult);
+ } catch (MDStoreServiceException e) {
+ throw new ServiceFacadeException("delivering records for md store " + mdStoreId + " failed!", e);
+ }
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.java
new file mode 100644
index 0000000000..00bb0c3f74
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.java
@@ -0,0 +1,45 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_MDSTORE_SERVICE_LOCATION;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_PAGESIZE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_PAGESIZE_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_READ_TIMEOUT_DEFAULT_VALUE;
+
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+import eu.dnetlib.dhp.common.WorkflowRuntimeParameters;
+
+/**
+ * WebService MDStore service facade factory.
+ *
+ * @author mhorst
+ *
+ */
+public class WebServiceMDStoreFacadeFactory implements ServiceFacadeFactory {
+
+
+ //------------------------ LOGIC --------------------------
+
+ @Override
+ public WebServiceMDStoreFacade instantiate(Map parameters) {
+ Preconditions.checkArgument(parameters.containsKey(IMPORT_MDSTORE_SERVICE_LOCATION),
+ "unknown MDStore service location: no parameter provided: '%s'", IMPORT_MDSTORE_SERVICE_LOCATION);
+
+ return new WebServiceMDStoreFacade(parameters.get(IMPORT_MDSTORE_SERVICE_LOCATION),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_READ_TIMEOUT, DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT, DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT, RESULTSET_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT, RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Integer.parseInt(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_PAGESIZE, RESULTSET_PAGESIZE_DEFAULT_VALUE, parameters)));
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.java
new file mode 100644
index 0000000000..6e1aee80b5
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.java
@@ -0,0 +1,52 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.enabling.tools.JaxwsServiceResolverImpl;
+
+/**
+ * WebService based ObjectStore facade.
+ *
+ * @author mhorst
+ *
+ */
+public class WebServiceObjectStoreFacade extends AbstractResultSetAwareWebServiceFacade implements ObjectStoreFacade {
+
+
+ //------------------------ CONSTRUCTORS -------------------
+
+ /**
+ * @param serviceLocation ObjectStore webservice location
+ * @param serviceReadTimeout service read timeout
+ * @param serviceConnectionTimeout service connection timeout
+ * @param resultSetReadTimeout resultset read timeout
+ * @param resultSetConnectionTimeout result set connection timeout
+ * @param resultSetPageSize resultset page size
+ */
+ public WebServiceObjectStoreFacade(String serviceLocation,
+ long serviceReadTimeout, long serviceConnectionTimeout,
+ long resultSetReadTimeout, long resultSetConnectionTimeout, int resultSetPageSize) {
+ super(ObjectStoreService.class, serviceLocation,
+ serviceReadTimeout, serviceConnectionTimeout,
+ resultSetReadTimeout, resultSetConnectionTimeout, resultSetPageSize);
+ }
+
+ //------------------------ LOGIC --------------------------
+
+ @Override
+ public Iterable deliverObjects(String objectStoreId, long from, long until) throws ServiceFacadeException {
+ try {
+ W3CEndpointReference eprResult = getService().deliverObjects(objectStoreId, from, until);
+ ResultSetClientFactory rsFactory = new ResultSetClientFactory(
+ getResultSetPageSize(), getResultSetReadTimeout(), getResultSetConnectionTimeout());
+ rsFactory.setServiceResolver(new JaxwsServiceResolverImpl());
+ return rsFactory.getClient(eprResult);
+ } catch (ObjectStoreServiceException e) {
+ throw new ServiceFacadeException("delivering records for object store " + objectStoreId + " failed!", e);
+ }
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.java
new file mode 100644
index 0000000000..9c77c45464
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.java
@@ -0,0 +1,44 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_CONTENT_OBJECT_STORE_LOC;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_PAGESIZE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_PAGESIZE_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_READ_TIMEOUT_DEFAULT_VALUE;
+
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+import eu.dnetlib.dhp.common.WorkflowRuntimeParameters;
+
+/**
+ * WebService ObjectStore facade factory.
+ *
+ * @author mhorst
+ *
+ */
+public class WebServiceObjectStoreFacadeFactory implements ServiceFacadeFactory {
+
+ //------------------------ LOGIC --------------------------
+
+ @Override
+ public WebServiceObjectStoreFacade instantiate(Map parameters) {
+ Preconditions.checkArgument(parameters.containsKey(IMPORT_CONTENT_OBJECT_STORE_LOC),
+ "unknown object store service location: no parameter provided: '%s'", IMPORT_CONTENT_OBJECT_STORE_LOC);
+
+ return new WebServiceObjectStoreFacade(parameters.get(IMPORT_CONTENT_OBJECT_STORE_LOC),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_READ_TIMEOUT, DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT, DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT, RESULTSET_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT, RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+ Integer.parseInt(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_PAGESIZE, RESULTSET_PAGESIZE_DEFAULT_VALUE, parameters)));
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.java
new file mode 100644
index 0000000000..5d61f06a5c
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.java
@@ -0,0 +1,94 @@
+package eu.dnetlib.dhp.wf.importer.mdrecord;
+
+import java.util.Stack;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import eu.dnetlib.dhp.common.InfoSpaceConstants;
+
+/**
+ * MDRecord handler extracting record identifier.
+ *
+ * Notice: writer is not being closed by handler. Created outside, let it be closed outside as well.
+ * @author mhorst
+ *
+ */
+public class MDRecordHandler extends DefaultHandler {
+
+ public static final String ELEM_OBJ_IDENTIFIER = "objIdentifier";
+
+ private static final String ELEM_HEADER = "header";
+
+ private Stack parents;
+
+ private StringBuilder currentValue = new StringBuilder();
+
+ private String recordId;
+
+
+ // ------------------------ LOGIC --------------------------
+
+ @Override
+ public void startDocument() throws SAXException {
+ parents = new Stack();
+ recordId = null;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String qName,
+ Attributes attributes) throws SAXException {
+ if (this.recordId == null) {
+ if (isWithinElement(localName, ELEM_OBJ_IDENTIFIER, ELEM_HEADER)) {
+// identifierType attribute is mandatory
+ this.currentValue = new StringBuilder();
+ }
+ this.parents.push(localName);
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String qName)
+ throws SAXException {
+ if (this.recordId == null) {
+ this.parents.pop();
+ if (isWithinElement(localName, ELEM_OBJ_IDENTIFIER, ELEM_HEADER)) {
+ this.recordId = InfoSpaceConstants.ROW_PREFIX_RESULT + this.currentValue.toString().trim();
+ }
+// resetting current value;
+ this.currentValue = null;
+ }
+ }
+
+ @Override
+ public void endDocument() throws SAXException {
+ parents.clear();
+ parents = null;
+ }
+
+ @Override
+ public void characters(char[] ch, int start, int length)
+ throws SAXException {
+ if (this.currentValue!=null) {
+ this.currentValue.append(ch, start, length);
+ }
+ }
+
+ /**
+ * @return record identifier
+ */
+ public String getRecordId() {
+ return recordId;
+ }
+
+ // ------------------------ PRIVATE --------------------------
+
+ private boolean isWithinElement(String localName, String expectedElement, String expectedParent) {
+ return localName.equals(expectedElement) && !this.parents.isEmpty() &&
+ expectedParent.equals(this.parents.peek());
+ }
+
+
+}
+
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.java
new file mode 100644
index 0000000000..4610938514
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.java
@@ -0,0 +1,157 @@
+package eu.dnetlib.dhp.wf.importer.mdrecord;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import com.google.common.base.Preconditions;
+import eu.dnetlib.dhp.common.WorkflowRuntimeParameters;
+import eu.dnetlib.dhp.common.counter.NamedCounters;
+import eu.dnetlib.dhp.common.counter.NamedCountersFileWriter;
+import eu.dnetlib.dhp.common.java.PortBindings;
+import eu.dnetlib.dhp.common.java.Process;
+import eu.dnetlib.dhp.common.java.io.DataStore;
+import eu.dnetlib.dhp.common.java.io.FileSystemPath;
+import eu.dnetlib.dhp.common.java.porttype.AvroPortType;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+import eu.dnetlib.dhp.importer.schemas.ImportedRecord;
+import eu.dnetlib.dhp.importer.schemas.RecordFormat;
+import eu.dnetlib.dhp.wf.importer.facade.MDStoreFacade;
+import eu.dnetlib.dhp.wf.importer.facade.ServiceFacadeUtils;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.log4j.Logger;
+import org.xml.sax.InputSource;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.OOZIE_ACTION_OUTPUT_FILENAME;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_MDSTORE_IDS_CSV;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_MDSTORE_RECORD_MAXLENGTH;
+
+/**
+ * {@link MDStoreFacade} based metadata records importer.
+ * @author mhorst
+ *
+ */
+public class MDStoreRecordsImporter implements Process {
+
+ protected static final String COUNTER_NAME_TOTAL = "TOTAL";
+
+ protected static final String COUNTER_NAME_SIZE_EXCEEDED = "SIZE_EXCEEDED";
+
+ protected static final String PORT_OUT_MDRECORDS = "mdrecords";
+
+ private static final Logger log = Logger.getLogger(MDStoreRecordsImporter.class);
+
+ private final static int progressLogInterval = 100000;
+
+ private final NamedCountersFileWriter countersWriter = new NamedCountersFileWriter();
+
+ private final Map outputPorts = new HashMap();
+
+
+ //------------------------ CONSTRUCTORS -------------------
+
+ public MDStoreRecordsImporter() {
+ outputPorts.put(PORT_OUT_MDRECORDS, new AvroPortType(ImportedRecord.SCHEMA$));
+ }
+
+ //------------------------ LOGIC --------------------------
+
+ @Override
+ public Map getInputPorts() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Map getOutputPorts() {
+ return outputPorts;
+ }
+
+ @Override
+ public void run(PortBindings portBindings, Configuration conf,
+ Map parameters) throws Exception {
+
+ Preconditions.checkArgument(parameters.containsKey(IMPORT_MDSTORE_IDS_CSV),
+ "unknown mdstore identifier, required parameter '%s' is missing!", IMPORT_MDSTORE_IDS_CSV);
+ String mdStoreIdsCSV = parameters.get(IMPORT_MDSTORE_IDS_CSV);
+ int recordMaxLength = parameters.containsKey(IMPORT_MDSTORE_RECORD_MAXLENGTH)?
+ Integer.parseInt(parameters.get(IMPORT_MDSTORE_RECORD_MAXLENGTH)):Integer.MAX_VALUE;
+
+ NamedCounters counters = new NamedCounters(new String[] { COUNTER_NAME_TOTAL, COUNTER_NAME_SIZE_EXCEEDED });
+
+ if (StringUtils.isNotBlank(mdStoreIdsCSV) && !WorkflowRuntimeParameters.UNDEFINED_NONEMPTY_VALUE.equals(mdStoreIdsCSV)) {
+
+ String[] mdStoreIds = StringUtils.split(mdStoreIdsCSV, WorkflowRuntimeParameters.DEFAULT_CSV_DELIMITER);
+
+ try (DataFileWriter recordWriter = getWriter(FileSystem.get(conf), portBindings)) {
+
+ MDStoreFacade mdStoreFacade = ServiceFacadeUtils.instantiate(parameters);
+
+ SAXParserFactory parserFactory = SAXParserFactory.newInstance();
+ parserFactory.setNamespaceAware(true);
+ SAXParser saxParser = parserFactory.newSAXParser();
+ MDRecordHandler mdRecordHandler = new MDRecordHandler();
+
+ long startTime = System.currentTimeMillis();
+ int currentCount = 0;
+
+ for (String mdStoreId : mdStoreIds) {
+ for (String mdRecord : mdStoreFacade.deliverMDRecords(mdStoreId)) {
+ if (!StringUtils.isEmpty(mdRecord)) {
+ if (mdRecord.length() <= recordMaxLength) {
+ saxParser.parse(new InputSource(new StringReader(mdRecord)), mdRecordHandler);
+ String recordId = mdRecordHandler.getRecordId();
+ if (StringUtils.isNotBlank(recordId)) {
+ recordWriter.append(
+ ImportedRecord.newBuilder()
+ .setId(recordId)
+ .setBody(mdRecord)
+ .setFormat(RecordFormat.XML)
+ .build());
+ counters.increment(COUNTER_NAME_TOTAL);
+ } else {
+ log.error("skipping, unable to extract identifier from record: " + mdRecord);
+ }
+ } else {
+ counters.increment(COUNTER_NAME_SIZE_EXCEEDED);
+ log.error("mdstore record maximum length (" + recordMaxLength + "): was exceeded: "
+ + mdRecord.length() + ", record content:\n" + mdRecord);
+ }
+
+ } else {
+ log.error("got empty metadata record from mdstore: " + mdStoreId);
+ }
+ currentCount++;
+ if (currentCount % progressLogInterval == 0) {
+ log.info("current progress: " + currentCount + ", last package of " + progressLogInterval
+ + " processed in " + ((System.currentTimeMillis() - startTime) / 1000) + " secs");
+ startTime = System.currentTimeMillis();
+ }
+ }
+ }
+ log.info("total number of processed records: " + currentCount);
+ }
+ }
+
+ if (counters.currentValue(COUNTER_NAME_TOTAL)==0) {
+ log.warn("parsed 0 metadata records from mdstores: " + mdStoreIdsCSV);
+ }
+ countersWriter.writeCounters(counters, System.getProperty(OOZIE_ACTION_OUTPUT_FILENAME));
+
+ }
+
+ /**
+ * Provides {@link ImportedRecord} writer consuming records.
+ */
+ protected DataFileWriter getWriter(FileSystem fs, PortBindings portBindings) throws IOException {
+ return DataStore.create(
+ new FileSystemPath(fs, portBindings.getOutput().get(PORT_OUT_MDRECORDS)), ImportedRecord.SCHEMA$);
+ }
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.java b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.java
new file mode 100644
index 0000000000..4776af22b0
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.java
@@ -0,0 +1,48 @@
+package eu.dnetlib.dhp.wf.importer.mdrecord;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import eu.dnetlib.dhp.common.java.PortBindings;
+import eu.dnetlib.dhp.common.java.Process;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+import org.apache.hadoop.conf.Configuration;
+
+public class MongoRecordImporter implements Process {
+
+ private final Map outputPorts = new HashMap();
+
+ @Override
+ public Map getInputPorts() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Map getOutputPorts() {
+ return outputPorts;
+ }
+
+ @Override
+ public void run(final PortBindings portBindings, final Configuration conf, final Map parameters) throws Exception {
+
+ /*
+ SparkSession spark = SparkSession.builder()
+ .master("local")
+ .appName("MongoSparkConnectorIntro")
+ .config("spark.mongodb.input.uri", "mongodb://127.0.0.1/test.myCollection")
+ .config("spark.mongodb.output.uri", "mongodb://127.0.0.1/test.myCollection")
+ .getOrCreate();
+
+ // Create a JavaSparkContext using the SparkSession's SparkContext object
+ JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());
+
+ // More application logic would go here...
+
+ jsc.close();
+ */
+
+ }
+
+
+}
diff --git a/dhp-wf/dhp-wf-import/src/main/resources/eu/dnetlib/dhp/wf/importer/mdrecord/oozie_app/workflow.xml b/dhp-wf/dhp-wf-import/src/main/resources/eu/dnetlib/dhp/wf/importer/mdrecord/oozie_app/workflow.xml
new file mode 100644
index 0000000000..dee61af910
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/src/main/resources/eu/dnetlib/dhp/wf/importer/mdrecord/oozie_app/workflow.xml
@@ -0,0 +1,124 @@
+
+
+
+
+ mdstore_facade_factory_classname
+ eu.dnetlib.dhp.wf.importer.facade.WebServiceMDStoreFacadeFactory
+ ServiceFacadeFactory implementation class name producing eu.dnetlib.dhp.wf.importer.facade.WebServiceMDStoreFacade
+
+
+ mdstore_service_location
+ $UNDEFINED$
+ mdstore service location
+
+
+ mdstore_ids_csv
+ $UNDEFINED$
+ comma separated mdstore identifiers
+
+
+ mdstore_record_maxlength
+ 500000
+ maximum allowed length of mdstore record
+
+
+ output
+ ImportedRecord avro datastore output holding mdrecords
+
+
+ output_report_root_path
+ base directory for storing reports
+
+
+ output_report_relative_path
+ import_mdrecord
+ directory for storing report (relative to output_report_root_path)
+
+
+ dnet_service_client_read_timeout
+ 60000
+ DNet service client reading timeout (expressed in milliseconds)
+
+
+ dnet_service_client_connection_timeout
+ 60000
+ DNet service client connection timeout (expressed in milliseconds)
+
+
+ resultset_client_read_timeout
+ 60000
+ result set client reading timeout (expressed in milliseconds)
+
+
+ resultset_client_connection_timeout
+ 60000
+ result set client connection timeout (expressed in milliseconds)
+
+
+ report_properties_prefix
+ import.mdrecord
+ report entry related to total number of imported records
+
+
+
+
+ ${jobTracker}
+ ${nameNode}
+
+
+ mapreduce.job.queuename
+ ${queueName}
+
+
+ oozie.launcher.mapred.job.queue.name
+ ${oozieLauncherQueueName}
+
+
+
+
+
+
+
+
+
+
+
+
+ eu.dnetlib.dhp.common.java.ProcessWrapper
+ eu.dnetlib.dhp.wf.importer.mdrecord.MDStoreRecordsImporter
+
+ -Pimport.mdstore.service.location=${mdstore_service_location}
+ -Pimport.mdstore.ids.csv=${mdstore_ids_csv}
+ -Pimport.mdstore.record.maxlength=${mdstore_record_maxlength}
+
+ -Pimport.resultset.client.read.timeout=${resultset_client_read_timeout}
+ -Pimport.resultset.client.connection.timeout=${resultset_client_connection_timeout}
+ -Pdnet.service.client.read.timeout=${dnet_service_client_read_timeout}
+ -Pdnet.service.client.connection.timeout=${dnet_service_client_connection_timeout}
+
+ -Pimport.facade.factory.classname=${mdstore_facade_factory_classname}
+ -Omdrecords=${output}
+
+
+
+
+
+
+
+
+ eu.dnetlib.dhp.common.java.ProcessWrapper
+ eu.dnetlib.dhp.common.report.ReportGenerator
+ -Preport.${report_properties_prefix}.total=${wf:actionData('mdrecord-importer')['TOTAL']}
+ -Preport.${report_properties_prefix}.invalid.sizeExceeded=${wf:actionData('mdrecord-importer')['SIZE_EXCEEDED']}
+ -Oreport=${output_report_root_path}/${output_report_relative_path}
+
+
+
+
+
+
+ Unfortunately, the process failed -- error message:
+ [${wf:errorMessage(wf:lastErrorNode())}]
+
+
+
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.class
new file mode 100644
index 0000000000..5bd02d3603
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.class
new file mode 100644
index 0000000000..46f329580a
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.class
new file mode 100644
index 0000000000..13a46a67ad
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/RecordReceiver.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/RecordReceiver.class
new file mode 100644
index 0000000000..fb8394cb5a
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/RecordReceiver.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.class
new file mode 100644
index 0000000000..abb39ad860
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.class
new file mode 100644
index 0000000000..7689db9d69
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.class
new file mode 100644
index 0000000000..ec0403005b
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.class
new file mode 100644
index 0000000000..e9e7a2cbdf
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.class
new file mode 100644
index 0000000000..03ce390c2d
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.class
new file mode 100644
index 0000000000..18e211bf9c
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.class
new file mode 100644
index 0000000000..d7663b0dd8
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.class
new file mode 100644
index 0000000000..aff40623b0
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.class
new file mode 100644
index 0000000000..c9ac132382
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.class
new file mode 100644
index 0000000000..cbd0403cbe
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.class
new file mode 100644
index 0000000000..b7c9aaee6c
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.class
new file mode 100644
index 0000000000..8ffb1d81bb
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.class
new file mode 100644
index 0000000000..b6a1ceca93
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.class
new file mode 100644
index 0000000000..092c87c701
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.class
new file mode 100644
index 0000000000..254a7f6478
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.class b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.class
new file mode 100644
index 0000000000..5079c03852
Binary files /dev/null and b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.class differ
diff --git a/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/oozie_app/workflow.xml b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/oozie_app/workflow.xml
new file mode 100644
index 0000000000..dee61af910
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/target/classes/eu/dnetlib/dhp/wf/importer/mdrecord/oozie_app/workflow.xml
@@ -0,0 +1,124 @@
+
+
+
+
+ mdstore_facade_factory_classname
+ eu.dnetlib.dhp.wf.importer.facade.WebServiceMDStoreFacadeFactory
+ ServiceFacadeFactory implementation class name producing eu.dnetlib.dhp.wf.importer.facade.WebServiceMDStoreFacade
+
+
+ mdstore_service_location
+ $UNDEFINED$
+ mdstore service location
+
+
+ mdstore_ids_csv
+ $UNDEFINED$
+ comma separated mdstore identifiers
+
+
+ mdstore_record_maxlength
+ 500000
+ maximum allowed length of mdstore record
+
+
+ output
+ ImportedRecord avro datastore output holding mdrecords
+
+
+ output_report_root_path
+ base directory for storing reports
+
+
+ output_report_relative_path
+ import_mdrecord
+ directory for storing report (relative to output_report_root_path)
+
+
+ dnet_service_client_read_timeout
+ 60000
+ DNet service client reading timeout (expressed in milliseconds)
+
+
+ dnet_service_client_connection_timeout
+ 60000
+ DNet service client connection timeout (expressed in milliseconds)
+
+
+ resultset_client_read_timeout
+ 60000
+ result set client reading timeout (expressed in milliseconds)
+
+
+ resultset_client_connection_timeout
+ 60000
+ result set client connection timeout (expressed in milliseconds)
+
+
+ report_properties_prefix
+ import.mdrecord
+ report entry related to total number of imported records
+
+
+
+
+ ${jobTracker}
+ ${nameNode}
+
+
+ mapreduce.job.queuename
+ ${queueName}
+
+
+ oozie.launcher.mapred.job.queue.name
+ ${oozieLauncherQueueName}
+
+
+
+
+
+
+
+
+
+
+
+
+ eu.dnetlib.dhp.common.java.ProcessWrapper
+ eu.dnetlib.dhp.wf.importer.mdrecord.MDStoreRecordsImporter
+
+ -Pimport.mdstore.service.location=${mdstore_service_location}
+ -Pimport.mdstore.ids.csv=${mdstore_ids_csv}
+ -Pimport.mdstore.record.maxlength=${mdstore_record_maxlength}
+
+ -Pimport.resultset.client.read.timeout=${resultset_client_read_timeout}
+ -Pimport.resultset.client.connection.timeout=${resultset_client_connection_timeout}
+ -Pdnet.service.client.read.timeout=${dnet_service_client_read_timeout}
+ -Pdnet.service.client.connection.timeout=${dnet_service_client_connection_timeout}
+
+ -Pimport.facade.factory.classname=${mdstore_facade_factory_classname}
+ -Omdrecords=${output}
+
+
+
+
+
+
+
+
+ eu.dnetlib.dhp.common.java.ProcessWrapper
+ eu.dnetlib.dhp.common.report.ReportGenerator
+ -Preport.${report_properties_prefix}.total=${wf:actionData('mdrecord-importer')['TOTAL']}
+ -Preport.${report_properties_prefix}.invalid.sizeExceeded=${wf:actionData('mdrecord-importer')['SIZE_EXCEEDED']}
+ -Oreport=${output_report_root_path}/${output_report_relative_path}
+
+
+
+
+
+
+ Unfortunately, the process failed -- error message:
+ [${wf:errorMessage(wf:lastErrorNode())}]
+
+
+
diff --git a/dhp-wf/dhp-wf-import/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/dhp-wf/dhp-wf-import/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst
new file mode 100644
index 0000000000..672248f223
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst
@@ -0,0 +1,20 @@
+eu/dnetlib/dhp/wf/importer/RecordReceiver.class
+eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.class
+eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.class
+eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.class
+eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.class
+eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.class
+eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.class
+eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.class
+eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.class
+eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.class
+eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.class
+eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.class
+eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.class
+eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.class
+eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.class
+eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.class
+eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.class
+eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.class
+eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.class
+eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.class
diff --git a/dhp-wf/dhp-wf-import/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/dhp-wf/dhp-wf-import/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst
new file mode 100644
index 0000000000..e9820d1d98
--- /dev/null
+++ b/dhp-wf/dhp-wf-import/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst
@@ -0,0 +1,20 @@
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/RecordReceiver.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.java
+/Users/claudio/workspace/dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.java
diff --git a/dhp-wf/pom.xml b/dhp-wf/pom.xml
new file mode 100644
index 0000000000..4c0aa666ac
--- /dev/null
+++ b/dhp-wf/pom.xml
@@ -0,0 +1,249 @@
+
+
+ 4.0.0
+
+
+ eu.dnetlib.dhp
+ dhp
+ 1.0.0-SNAPSHOT
+
+
+ dhp-wf
+ pom
+
+
+ dhp-wf-import
+
+
+
+ yyyy-MM-dd_HH_mm
+
+
+ oozie-package
+
+ src/test/resources/define/path/pointing/to/directory/holding/oozie_app
+ oozie_app
+ default
+ default
+ default
+ primed
+
+ runtime
+
+ true
+
+ ${user.home}/.dhp/application.properties
+
+ ${maven.build.timestamp}
+
+ ${project.version}
+ true
+
+
+
+
+
+
+ org.apache.oozie
+ oozie-client
+
+
+ net.schmizz
+ sshj
+ test
+
+
+
+
+
+ oozie-package
+
+
+
+ org.apache.maven.plugins
+ maven-enforcer-plugin
+ 1.4.1
+
+
+ enforce-connection-properties-file-existence
+ initialize
+
+ enforce
+
+
+
+
+
+ ${dhpConnectionProperties}
+
+
+ The file with connection properties could not be found. Please, create the ${dhpConnectionProperties} file or set the location to another already created file by using
+ -DdhpConnectionProperties property.
+
+
+
+ true
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+
+ copy dependencies
+ prepare-package
+
+ copy-dependencies
+
+
+ ${oozie.package.dependencies.include.scope}
+ ${oozie.package.dependencies.exclude.scope}
+ true
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+ attach-test-resources-package
+ prepare-package
+
+ test-jar
+
+
+ ${oozie.package.skip.test.jar}
+
+
+
+
+
+
+
+ pl.project13.maven
+ git-commit-id-plugin
+ 2.1.11
+
+
+
+ revision
+
+
+
+
+ true
+ yyyy-MM-dd'T'HH:mm:ssZ
+ true
+ target/${oozie.package.file.name}/${oozieAppDir}/version.properties
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+ 3.0.0
+
+
+ assembly-oozie-installer
+ package
+
+ single
+
+
+ false
+ ${oozie.package.file.name}_shell_scripts
+
+ oozie-installer
+
+
+
+
+
+
+
+
+
+ maven-antrun-plugin
+
+
+
+ installer-copy-custom
+ process-resources
+
+ run
+
+
+
+
+
+
+
+
+
+
+ package
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ run
+
+
+
+
+
+
+
+
+
+
+
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000000..8861cbe25f
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,953 @@
+
+
+ 4.0.0
+
+ eu.dnetlib.dhp
+ dhp
+ 1.0.0-SNAPSHOT
+ pom
+
+ http://www.d-net.research-infrastructures.eu
+
+
+
+ The Apache Software License, Version 2.0
+ http://www.apache.org/licenses/LICENSE-2.0.txt
+ repo
+ A business-friendly OSS license
+
+
+
+
+ dhp-build
+ dhp-common
+ dhp-schemas
+ dhp-wf
+
+
+
+ Redmine
+ https://issue.openaire.research-infrastructures.eu/projects/openaire
+
+
+
+ jenkins
+ https://jenkins-dnet.d4science.org/
+
+
+
+ scm:git:ssh://git@github.com/???
+ scm:git:ssh://git@github.com/???.git
+ https://github.com/???
+ HEAD
+
+
+
+
+
+
+
+
+ dnet45-releases
+ D-Net 45 releases
+ http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-releases
+ default
+
+ false
+
+
+ true
+
+
+
+ dnet45-bootstrap-release
+ dnet45 bootstrap release
+ http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-bootstrap-release
+ default
+
+ false
+
+
+ true
+
+
+
+
+ cloudera
+ Cloudera Repository
+ https://repository.cloudera.com/artifactory/cloudera-repos
+
+ true
+
+
+ false
+
+
+
+
+
+
+
+ org.slf4j
+ slf4j-api
+ 1.7.22
+
+
+ org.slf4j
+ slf4j-log4j12
+ 1.7.22
+
+
+ log4j
+ log4j
+ 1.2.17
+
+
+ javax.servlet
+ javax.servlet-api
+ 3.1.0
+ runtime
+
+
+ junit
+ junit
+ 4.12
+ test
+
+
+ org.hamcrest
+ hamcrest-all
+ 1.3
+ test
+
+
+ org.mockito
+ mockito-all
+ 1.10.19
+ test
+
+
+ org.powermock
+ powermock-core
+ 1.6.6
+ test
+
+
+ com.google.code.findbugs
+ annotations
+ 3.0.1
+ provided
+
+
+ com.google.code.findbugs
+ jsr305
+ 3.0.1
+ provided
+
+
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ ${dhp.hadoop.version}
+ provided
+
+
+ servlet-api
+ javax.servlet
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-common
+ ${dhp.hadoop.version}
+ provided
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-common
+ ${dhp.hadoop.version}
+ provided
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-common
+ ${dhp.hadoop.version}
+ test
+ test-jar
+
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-common
+ ${dhp.hadoop.version}
+ test
+ test-jar
+
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-app
+ ${dhp.hadoop.version}
+ provided
+
+
+ servlet-api
+ javax.servlet
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+ ${dhp.hadoop.version}
+ provided
+
+
+ servlet-api
+ javax.servlet
+
+
+
+ org.apache.calcite
+ calcite-core
+
+
+ org.apache.calcite
+ calcite-avatica
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ ${dhp.hadoop.version}
+
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ ${dhp.hadoop.version}
+ test-jar
+ test
+
+
+
+
+
+ org.apache.oozie
+ oozie-core
+ ${dhp.oozie.version}
+
+
+
+ servlet-api
+ javax.servlet
+
+
+
+ org.apache.calcite
+ calcite-core
+
+
+ org.apache.calcite
+ calcite-avatica
+
+
+
+
+
+ org.apache.oozie
+ oozie-client
+ ${dhp.oozie.version}
+
+
+
+ slf4j-simple
+ org.slf4j
+
+
+
+
+
+ org.apache.spark
+ spark-core_2.10
+ ${dhp.spark.version}
+ provided
+
+
+
+ org.apache.spark
+ spark-sql_2.10
+ ${dhp.spark.version}
+ provided
+
+
+
+ com.databricks
+ spark-avro_2.10
+ 1.1.0-${dhp.cdh.version}
+
+
+
+ com.databricks
+ spark-csv_2.10
+ 1.5.0
+
+
+
+ pl.edu.icm.spark-utils
+ spark-utils
+ 1.0.0
+
+
+
+ org.mongodb.spark
+ mongo-spark-connector_2.10
+ 2.2.1
+ provided
+
+
+
+
+
+ org.apache.avro
+ avro
+ ${dhp.avro.version}
+
+
+
+ org.apache.avro
+ avro-mapred
+ ${dhp.avro.version}
+ hadoop2
+
+
+
+ servlet-api
+ org.mortbay.jetty
+
+
+ netty
+ io.netty
+
+
+
+
+
+
+
+ org.apache.pig
+ pig
+ ${dhp.pig.version}
+ provided
+
+
+ org.mortbay.jetty
+ servlet-api-2.5
+
+
+ servlet-api
+ javax.servlet
+
+
+
+
+
+ org.apache.pig
+ piggybank
+ ${dhp.pig.version}
+ provided
+
+
+
+ org.apache.pig
+ pigunit
+ ${dhp.pig.version}
+
+
+
+
+ pl.edu.icm.cermine
+ cermine-impl
+ 1.13
+
+
+
+
+ pl.edu.icm.coansys
+ models
+ ${dhp.coansys.version}
+
+
+
+ pl.edu.icm.coansys
+ citation-matching-core-code
+ ${dhp.coansys.version}
+
+
+
+ org.apache.avro
+ avro-mapred
+
+
+ org.apache.hadoop
+ hadoop-client
+
+
+ org.apache.hadoop
+ hadoop-yarn-api
+
+
+ org.apache.hadoop
+ hadoop-yarn-client
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-common
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-app
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+
+
+
+ org.slf4j
+ slf4j-simple
+
+
+
+
+
+
+
+ eu.dnetlib
+ dnet-openaireplus-mapping-utils
+ [6.0.0, 7.0.0)
+
+
+
+ org.apache.hadoop
+ hadoop-core
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+
+
+ org.apache.zookeeper
+ zookeeper
+
+
+ jgrapht
+ jgrapht
+
+
+
+
+
+ eu.dnetlib
+ dnet-objectstore-rmi
+ [2.0.0, 3.0.0)
+
+
+
+ eu.dnetlib
+ cnr-rmi-api
+ [2.0.0, 3.0.0)
+
+
+
+ eu.dnetlib
+ cnr-resultset-client
+ [2.0.0, 3.0.0)
+
+
+
+ org.springframework
+ spring-web
+
+
+ org.springframework
+ spring-webmvc
+
+
+
+
+
+ eu.dnetlib
+ dnet-actionmanager-common
+ [6.0.0, 7.0.0)
+
+
+
+ apache
+ commons-logging
+
+
+
+
+
+ eu.dnetlib
+ cnr-service-utils
+ [1.0.0, 2.0.0)
+
+
+
+
+ com.beust
+ jcommander
+ 1.60
+
+
+
+ com.google.code.gson
+ gson
+ 2.8.0
+
+
+
+ com.googlecode.json-simple
+ json-simple
+ 1.1.1
+
+
+
+ org.apache.commons
+ commons-lang3
+ 3.5
+
+
+
+ org.apache.commons
+ commons-collections4
+ 4.1
+
+
+
+ com.thoughtworks.xstream
+ xstream
+ 1.4.9
+
+
+
+ xalan
+ xalan
+ 2.7.2
+
+
+
+ xml-apis
+ xml-apis
+ 1.4.01
+
+
+
+ org.jdom
+ jdom
+ 1.1.3
+
+
+
+ org.jsoup
+ jsoup
+ 1.10.2
+
+
+
+ net.sf.opencsv
+ opencsv
+ 2.3
+
+
+
+ com.googlecode.protobuf-java-format
+ protobuf-java-format
+ 1.2
+
+
+
+ com.google.protobuf
+ protobuf-java
+ 2.5.0
+
+
+
+
+ com.google.guava
+ guava
+ 12.0
+
+
+
+ commons-cli
+ commons-cli
+ 1.3.1
+
+
+
+ commons-io
+ commons-io
+ 2.5
+
+
+
+ de.sven-jacobs
+ loremipsum
+ 1.0
+
+
+
+ net.schmizz
+ sshj
+ 0.10.0
+
+
+
+
+ org.bouncycastle
+ bcprov-jdk15on
+ 1.50
+
+
+
+
+ com.thoughtworks.paranamer
+ paranamer
+ 2.8
+
+
+
+
+ com.linkedin.datafu
+ datafu
+ 1.2.0
+
+
+
+ org.apache.cxf
+ cxf-rt-frontend-jaxws
+ ${cxf.version}
+
+
+
+
+ com.sun.xml.bind
+ jaxb-impl
+ 2.2.7
+ runtime
+
+
+
+ org.springframework
+ spring-beans
+ ${spring.version}
+
+
+
+ org.springframework
+ spring-context
+ ${spring.version}
+
+
+
+ org.scala-lang
+ scala-library
+ ${scala.version}
+
+
+
+ commons-beanutils
+ commons-beanutils
+ 1.9.3
+
+
+
+ org.apache.curator
+ curator-test
+ 3.3.0
+ test
+
+
+
+
+
+
+
+ target
+ target/classes
+ ${project.artifactId}-${project.version}
+ target/test-classes
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.6.0
+
+ 1.8
+ 1.8
+ ${project.build.sourceEncoding}
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.0.2
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+ 3.0.1
+
+
+ attach-sources
+ verify
+
+ jar-no-fork
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+ 2.19.1
+
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+ 2.10.4
+
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+ 3.0.0
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+ 3.2.2
+
+
+ scala-compile-first
+ process-resources
+
+ add-source
+ compile
+
+
+
+ scala-test-compile
+ process-test-resources
+
+ testCompile
+
+
+
+
+
+
+
+ org.apache.avro
+ avro-maven-plugin
+ 1.7.7
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+ 1.12
+
+
+
+
+ org.eclipse.m2e
+ lifecycle-mapping
+ 1.0.0
+
+
+
+
+
+
+ org.apache.avro
+
+
+ avro-maven-plugin
+
+
+ [1.7.4,)
+
+
+ idl-protocol
+ schema
+
+
+
+
+
+
+
+
+
+ org.codehaus.mojo
+
+
+ build-helper-maven-plugin
+
+
+ [1.7,)
+
+
+ add-source
+
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+
+
+ maven-plugin-plugin
+
+
+ [3.2,)
+
+
+ descriptor
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-release-plugin
+ 2.5.3
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+ 0.7.9
+
+
+ **/schemas/*
+ **/com/cloudera/**/*
+ **/org/apache/avro/io/**/*
+
+
+
+
+ default-prepare-agent
+
+ prepare-agent
+
+
+
+ default-report
+ prepare-package
+
+ report
+
+
+
+
+
+
+
+
+ org.apache.maven.wagon
+ wagon-ssh
+ 2.10
+
+
+
+
+
+
+
+
+ dnet45-snapshots
+ DNet45 Snapshots
+ http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots
+ default
+
+
+ dnet45-releases
+ http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-releases
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+ 2.10.4
+
+ true
+
+
+
+
+
+
+ UTF-8
+ UTF-8
+ [4.2.5.RELEASE]
+ [3.1.5]
+
+ cdh5.9.0
+
+ 4.1.0-${dhp.cdh.version}
+ 0.12.0-${dhp.cdh.version}
+ 1.7.6-${dhp.cdh.version}
+ 2.6.0-${dhp.cdh.version}
+ 1.6.0-${dhp.cdh.version}
+ 2.10.6
+
+
+