dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/SparkGenEnrichedOrcidWorks.java

@@ -91,19 +91,18 @@ public class SparkGenEnrichedOrcidWorks {
 						Encoders.tuple(Encoders.STRING(), Encoders.STRING()))
 					.filter(Objects::nonNull)
 					.toJavaRDD();
-				logger.info("Works enriched data created: " + enrichedWorksRDD.count());
 				enrichedWorksRDD.saveAsTextFile(workingPath + outputEnrichedWorksPath);
 				logger.info("Works enriched data saved");
-				JavaRDD<Tuple2<String, Publication>> oafPublicationRDD = enrichedWorksRDD.map(e -> {
+				JavaRDD<Publication> oafPublicationRDD = enrichedWorksRDD.map(e -> {
 					JsonElement j = new JsonParser().parse(e._2());
-					return new Tuple2<>(e._1(), (Publication) PublicationToOaf
+					return (Publication) PublicationToOaf
-						.generatePublicationActionsFromDump(j.getAsJsonObject()));
+						.generatePublicationActionsFromDump(j.getAsJsonObject());
-				});
+				}).filter(p -> p != null);
 
-				Dataset<Tuple2<String, Publication>> publicationDataset = spark
+				Dataset<Publication> publicationDataset = spark
 					.createDataset(
 						oafPublicationRDD.repartition(1).rdd(),
-						Encoders.tuple(Encoders.STRING(), Encoders.bean(Publication.class)));
+						Encoders.bean(Publication.class));
 				publicationDataset.write().mode(SaveMode.Overwrite).save(workingPath + "no_doi_dataset/output");
 			});
 	}

dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/gen_enriched_orcid_works/oozie_app/config-default.xml

@@ -1,31 +0,0 @@
-<configuration>
-    <property>
-        <name>oozie.action.sharelib.for.java</name>
-        <value>spark2</value>
-    </property>
-    <property>
-        <name>oozie.launcher.mapreduce.user.classpath.first</name>
-        <value>true</value>
-    </property>
-    <property>
-        <name>oozie.launcher.mapreduce.map.java.opts</name>
-        <value>-Xmx4g</value>
-    </property>
-    <property>
-        <name>jobTracker</name>
-        <value>yarnRM</value>
-    </property>
-    <property>
-        <name>nameNode</name>
-        <value>hdfs://nameservice1</value>
-    </property>
-
-    <property>
-        <name>oozie.use.system.libpath</name>
-        <value>true</value>
-    </property>
-    <property>
-        <name>oozie.action.sharelib.for.spark</name>
-        <value>spark2</value>
-    </property>
-</configuration>

dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/gen_enriched_orcid_works/oozie_app/workflow.xml

@@ -1,11 +1,56 @@
-<workflow-app name="Gen Enriched Orcid Works" xmlns="uri:oozie:workflow:0.5">
+<workflow-app name="gen_orcid_no_doi_dataset" xmlns="uri:oozie:workflow:0.5">
     <parameters>
+        <property>
+            <name>sparkDriverMemory</name>
+            <description>memory for driver process</description>
+        </property>
+        <property>
+            <name>sparkExecutorMemory</name>
+            <description>memory for individual executor</description>
+        </property>
+        <property>
+            <name>sparkExecutorCores</name>
+            <description>number of cores used by single executor</description>
+        </property>
+        <property>
+            <name>oozieActionShareLibForSpark2</name>
+            <description>oozie action sharelib for spark 2.*</description>
+        </property>
+        <property>
+            <name>spark2ExtraListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
+            <description>spark 2.* extra listeners classname</description>
+        </property>
+        <property>
+            <name>spark2SqlQueryExecutionListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
+            <description>spark 2.* sql query execution listeners classname</description>
+        </property>
+        <property>
+            <name>spark2YarnHistoryServerAddress</name>
+            <description>spark 2.* yarn history server address</description>
+        </property>
+        <property>
+            <name>spark2EventLogDir</name>
+            <description>spark 2.* event log dir location</description>
+        </property>
         <property>
             <name>workingPath</name>
             <description>the working dir base path</description>
         </property>
     </parameters>
 
+    <global>
+        <job-tracker>${jobTracker}</job-tracker>
+        <name-node>${nameNode}</name-node>
+        <configuration>
+            <property>
+                <name>oozie.action.sharelib.for.spark</name>
+                <value>${oozieActionShareLibForSpark2}</value>
+            </property>
+        </configuration>
+    </global>
+
     <start to="ResetWorkingPath"/>
 
     <kill name="Kill">
@@ -16,20 +61,25 @@
         <fs>
             <delete path='${workingPath}/no_doi_enriched_works/output'/>
         </fs>
-        <ok to="Gen_Enriched_Orcid_Works"/>
+        <ok to="GenOrcidNoDoiDataset"/>
         <error to="Kill"/>
     </action>
 
-    <action name="Gen_Enriched_Orcid_Works">
+    <action name="GenOrcidNoDoiDataset">
         <spark xmlns="uri:oozie:spark-action:0.2">
-            <job-tracker>${jobTracker}</job-tracker>
+            <master>yarn-cluster</master>
-            <name-node>${nameNode}</name-node>
-            <master>yarn</master>
             <mode>cluster</mode>
-            <name>Gen_Enriched_Orcid_Works</name>
+            <name>GenOrcidNoDoiDataset</name>
             <class>eu.dnetlib.doiboost.orcidnodoi.SparkGenEnrichedOrcidWorks</class>
-            <jar>dhp-doiboost-1.2.4-SNAPSHOT.jar</jar>
+            <jar>dhp-doiboost-${projectVersion}.jar</jar>
-            <spark-opts>--num-executors 10 --conf spark.yarn.jars=&quot;hdfs://hadoop-rm1.garr-pa1.d4science.org:8020/user/oozie/share/lib/lib_20180405103059/spark2&quot; --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory}
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
             </spark-opts>
             <arg>-w</arg><arg>${workingPath}/</arg>
             <arg>-n</arg><arg>${nameNode}</arg>

dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/xml/XMLRecordParserTest.java

@@ -12,7 +12,7 @@ import eu.dnetlib.doiboost.orcid.model.WorkData;
 public class XMLRecordParserTest {
 
 	@Test
-	public void testOrcidAuthorDataXMLParser() throws Exception {
+	private void testOrcidAuthorDataXMLParser() throws Exception {
 
 		String xml = IOUtils.toString(this.getClass().getResourceAsStream("summary_0000-0001-6828-479X.xml"));
 
@@ -27,7 +27,7 @@ public class XMLRecordParserTest {
 	}
 
 	@Test
-	public void testOrcidXMLErrorRecordParser() throws Exception {
+	private void testOrcidXMLErrorRecordParser() throws Exception {
 
 		String xml = IOUtils.toString(this.getClass().getResourceAsStream("summary_error.xml"));
 
@@ -40,7 +40,7 @@ public class XMLRecordParserTest {
 	}
 
 	@Test
-	public void testOrcidWorkDataXMLParser() throws Exception {
+	private void testOrcidWorkDataXMLParser() throws Exception {
 
 		String xml = IOUtils
 			.toString(