merge upstream

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkUpdateEntity.java

@@ -13,6 +13,7 @@ import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.api.java.function.PairFunction;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
@@ -91,6 +92,7 @@ public class SparkUpdateEntity extends AbstractSparkAction {
 
 						final JavaPairRDD<String, String> mergedIds = rel
 							.where("relClass == 'merges'")
+							.where("source != target")
 							.select(rel.col("target"))
 							.distinct()
 							.toJavaRDD()

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/finalizedb.sh

@@ -10,6 +10,7 @@ export SOURCE=$1
 export SHADOW=$2
 
 echo "Updating shadow database"
+impala-shell -q "invalidate metadata"
 impala-shell -d ${SOURCE} -q "invalidate metadata"
 impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/^\(.*\)/compute stats ${SOURCE}.\1;/" | impala-shell -c -f -
 impala-shell -q "create database if not exists ${SHADOW}"