Merge branch 'master' of code-repo.d4science.org:D-Net/dnet-hadoop

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java

@@ -15,13 +15,8 @@ public class SparkGraphImporterJob {
 
         final ArgumentApplicationParser parser = new ArgumentApplicationParser(IOUtils.toString(SparkGraphImporterJob.class.getResourceAsStream("/eu/dnetlib/dhp/graph/input_graph_parameters.json")));
         parser.parseArgument(args);
-        final SparkSession spark = SparkSession
+
-                .builder()
+        try(SparkSession spark = getSparkSession(parser)) {
-                .appName(SparkGraphImporterJob.class.getSimpleName())
-                .master(parser.get("master"))
-                .config("hive.metastore.uris", parser.get("hive_metastore_uris"))
-                .enableHiveSupport()
-                .getOrCreate();
 
             final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
             final String inputPath = parser.get("sourcePath");
@@ -39,6 +34,16 @@ public class SparkGraphImporterJob {
                         .mode(SaveMode.Overwrite)
                         .saveAsTable(hiveDbName + "." + name);
             });
+        }
+    }
 
+    private static SparkSession getSparkSession(ArgumentApplicationParser parser) {
+        return SparkSession
+                .builder()
+                .appName(SparkGraphImporterJob.class.getSimpleName())
+                .master(parser.get("master"))
+                .config("hive.metastore.uris", parser.get("hive_metastore_uris"))
+                .enableHiveSupport()
+                .getOrCreate();
     }
 }

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/hive/postprocessing.sql

@@ -0,0 +1,8 @@
+CREATE view result as
+    select id, dateofcollection, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hive_db_name}.publication p
+    union all
+    select id, dateofcollection, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hive_db_name}.dataset d
+    union all
+    select id, dateofcollection, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hive_db_name}.software s
+    union all
+    select id, dateofcollection, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hive_db_name}.otherresearchproduct o;

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml

@@ -37,12 +37,29 @@
             <name>MapGraphIntoDataFrame</name>
             <class>eu.dnetlib.dhp.graph.SparkGraphImporterJob</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
-            <spark-opts>--executor-memory ${sparkExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory=${sparkDriverMemory} --conf spark.extraListeners="com.cloudera.spark.lineage.NavigatorAppListener" --conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener" --conf spark.sql.warehouse.dir="/user/hive/warehouse"</spark-opts>
+            <spark-opts>
+                --executor-memory ${sparkExecutorMemory}
+                --executor-cores ${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners="com.cloudera.spark.lineage.NavigatorAppListener"
+                --conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener"
+                --conf spark.sql.warehouse.dir="/user/hive/warehouse"</spark-opts>
             <arg>-mt</arg> <arg>yarn-cluster</arg>
             <arg>--sourcePath</arg><arg>${sourcePath}</arg>
             <arg>--hive_db_name</arg><arg>${hive_db_name}</arg>
             <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
         </spark>
+        <ok to="PostProcessing"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="PostProcessing">
+        <hive xmlns="uri:oozie:hive-action:0.2">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <script>/eu/dnetlib/dhp/graph/hive/postprocessing.sql</script>
+            <param>hive_db_name=${hive_db_name}</param>
+        </hive>
         <ok to="End"/>
         <error to="Kill"/>
     </action>

dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml

@@ -50,9 +50,9 @@
             <class>eu.dnetlib.dhp.graph.SparkXmlRecordBuilderJob</class>
             <jar>dhp-graph-provision-${projectVersion}.jar</jar>
             <spark-opts>
-                --executor-memory ${sparkExecutorMemory}
+                --executor-cores ${sparkExecutorCoresForJoining}
-                --executor-cores ${sparkExecutorCores}
+                --executor-memory ${sparkExecutorMemoryForJoining}
-                --driver-memory=${sparkDriverMemory}
+                --driver-memory=${sparkDriverMemoryForJoining}
                 --conf spark.dynamicAllocation.maxExecutors=${sparkExecutorCoresForJoining}
                 --conf spark.extraListeners="com.cloudera.spark.lineage.NavigatorAppListener"
                 --conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener"
@@ -79,8 +79,9 @@
             <class>eu.dnetlib.dhp.graph.SparkXmlIndexingJob</class>
             <jar>dhp-graph-provision-${projectVersion}.jar</jar>
             <spark-opts>
-                --executor-memory ${sparkExecutorMemory}
+                --executor-cores ${sparkExecutorCoresForIndexing}
-                --driver-memory=${sparkDriverMemory}
+                --executor-memory ${sparkExecutorMemoryForIndexing}
+                --driver-memory=${sparkDriverMemoryForIndexing}
                 --conf spark.dynamicAllocation.maxExecutors=${sparkExecutorCoresForIndexing}
                 --conf spark.extraListeners="com.cloudera.spark.lineage.NavigatorAppListener"
                 --conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener"