From 60bc2b1a20342c809a49f97b8703120708e9c5b7 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 27 Feb 2020 10:10:55 +0100 Subject: [PATCH] drop the hive DB before populating it from scratch --- .../main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java | 1 + .../java/eu/dnetlib/dhp/graph/model/SortableRelationKey.java | 4 ++++ .../java/eu/dnetlib/dhp/graph/utils/RelationPartitioner.java | 4 ++++ .../src/test/java/eu/dnetlib/dhp/graph/GraphJoinerTest.java | 4 ++++ 4 files changed, 13 insertions(+) create mode 100644 dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/graph/model/SortableRelationKey.java create mode 100644 dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/graph/utils/RelationPartitioner.java create mode 100644 dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/graph/GraphJoinerTest.java diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java index c5223c1f6..633558906 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/graph/SparkGraphImporterJob.java @@ -27,6 +27,7 @@ public class SparkGraphImporterJob { final String inputPath = parser.get("sourcePath"); final String hiveDbName = parser.get("hive_db_name"); + spark.sql(String.format("DROP DATABASE IF EXISTS %s CASCADE", hiveDbName)); spark.sql(String.format("CREATE DATABASE IF NOT EXISTS %s", hiveDbName)); // Read the input file and convert it into RDD of serializable object diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/graph/model/SortableRelationKey.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/graph/model/SortableRelationKey.java new file mode 100644 index 000000000..ea56c6d12 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/graph/model/SortableRelationKey.java @@ -0,0 +1,4 @@ +package eu.dnetlib.dhp.graph; + +public class SortableRelationKey { +} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/graph/utils/RelationPartitioner.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/graph/utils/RelationPartitioner.java new file mode 100644 index 000000000..0a7c6dcaf --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/graph/utils/RelationPartitioner.java @@ -0,0 +1,4 @@ +package eu.dnetlib.dhp.graph.utils; + +public class RelationPartitioner { +} diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/graph/GraphJoinerTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/graph/GraphJoinerTest.java new file mode 100644 index 000000000..e8c1c088c --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/graph/GraphJoinerTest.java @@ -0,0 +1,4 @@ +package eu.dnetlib.dhp.graph; + +public class GraphJoinerTest { +}