set configuration property in openorgs duplicates wf

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java

@@ -1,3 +1,4 @@
+
 package eu.dnetlib.dhp.oa.dedup;
 
 import java.io.IOException;
@@ -5,10 +6,7 @@ import java.util.Optional;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.FilterFunction;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.api.java.function.PairFunction;
 import org.apache.spark.sql.Dataset;
@@ -21,7 +19,6 @@ import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXException;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
-import eu.dnetlib.dhp.oa.dedup.model.Block;
 import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.Relation;
 import eu.dnetlib.dhp.utils.ISLookupClientFactory;
@@ -31,7 +28,6 @@ import eu.dnetlib.pace.config.DedupConfig;
 import eu.dnetlib.pace.model.MapDocument;
 import eu.dnetlib.pace.util.MapDocumentUtil;
 import scala.Tuple2;
-import scala.Tuple3;
 
 public class SparkWhitelistSimRels extends AbstractSparkAction {
 
@@ -82,7 +78,9 @@ public class SparkWhitelistSimRels extends AbstractSparkAction {
 		JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
 
 		// file format: source####target
-        Dataset<Tuple2<String, String>> whiteListRels = spark.createDataset(sc
+		Dataset<Tuple2<String, String>> whiteListRels = spark
+			.createDataset(
+				sc
 					.textFile(whiteListPath)
 					// check if the line is in the correct format: id1####id2
 					.filter(s -> s.contains(WHITELIST_SEPARATOR) && s.split(WHITELIST_SEPARATOR).length == 2)
@@ -99,7 +97,9 @@ public class SparkWhitelistSimRels extends AbstractSparkAction {
 
 			final String outputPath = DedupUtility.createSimRelPath(workingPath, actionSetId, subEntity);
 
-            Dataset<Tuple2<String, String>> entities = spark.createDataset(sc
+			Dataset<Tuple2<String, String>> entities = spark
+				.createDataset(
+					sc
 						.textFile(DedupUtility.createEntityPath(graphBasePath, subEntity))
 						.repartition(numPartitions)
 						.mapToPair(
@@ -112,17 +112,20 @@ public class SparkWhitelistSimRels extends AbstractSparkAction {
 
 			Dataset<Tuple2<String, String>> whiteListRels1 = whiteListRels
 				.joinWith(entities, whiteListRels.col("_1").equalTo(entities.col("_1")), "inner")
-                    .map((MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1, Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+				.map(
+					(MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1,
+					Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
 
 			Dataset<Tuple2<String, String>> whiteListRels2 = whiteListRels1
 				.joinWith(entities, whiteListRels1.col("_2").equalTo(entities.col("_1")), "inner")
-                    .map((MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1, Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+				.map(
+					(MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1,
+					Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
 
 			Dataset<Relation> whiteListSimRels = whiteListRels2
-                    .map((MapFunction<Tuple2<String, String>, Relation>)
+				.map(
-                                    r -> createSimRel(r._1(), r._2(), entity),
+					(MapFunction<Tuple2<String, String>, Relation>) r -> createSimRel(r._1(), r._2(), entity),
-                                    Encoders.bean(Relation.class)
+					Encoders.bean(Relation.class));
-                    );
 
 			saveParquet(whiteListSimRels, outputPath, SaveMode.Append);
 		}

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java

@@ -1,7 +1,8 @@
+
 package eu.dnetlib.dhp.oa.dedup;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.concurrent.TimeUnit;
-import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
@@ -10,7 +11,9 @@ import org.apache.http.impl.client.HttpClients;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 
 public class UpdateOpenorgsJob {
 
@@ -18,7 +21,10 @@ public class UpdateOpenorgsJob {
 
 	public static void main(String[] args) throws Exception {
 		ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                    IOUtils.toString(SparkCreateSimRels.class.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json")));
+			IOUtils
+				.toString(
+					SparkCreateSimRels.class
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json")));
 		parser.parseArgument(args);
 
 		final String apiUrl = parser.get("apiUrl");
@@ -107,9 +113,5 @@ class APIResponse {
 }
 
 enum ImportStatus {
-	SUCCESS,
+	SUCCESS, FAILED, RUNNING, NOT_LAUNCHED, NOT_YET_STARTED
-	FAILED,
-	RUNNING,
-	NOT_LAUNCHED,
-	NOT_YET_STARTED
 }

dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml

@@ -267,6 +267,12 @@
 
     <action name="update_openorgs">
         <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
             <main-class>eu.dnetlib.dhp.oa.dedup.UpdateOpenorgsJob</main-class>
             <arg>--apiUrl</arg><arg>${apiUrl}</arg>
             <arg>--delay</arg><arg>5</arg>

dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java

@@ -269,17 +269,23 @@ public class SparkDedupTest implements Serializable {
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "software"));
 
 		// check if the first relation in the whitelist exists
-        assertTrue(sw_simrel
+		assertTrue(
+			sw_simrel
 				.as(Encoders.bean(Relation.class))
 				.toJavaRDD()
-                .filter(rel ->
+				.filter(
-                        rel.getSource().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[0]) && rel.getTarget().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[1])).count() > 0);
+					rel -> rel.getSource().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[0])
+						&& rel.getTarget().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[1]))
+				.count() > 0);
 		// check if the second relation in the whitelist exists
-        assertTrue(sw_simrel
+		assertTrue(
+			sw_simrel
 				.as(Encoders.bean(Relation.class))
 				.toJavaRDD()
-                .filter(rel ->
+				.filter(
-                        rel.getSource().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[0]) && rel.getTarget().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[1])).count() > 0);
+					rel -> rel.getSource().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[0])
+						&& rel.getTarget().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[1]))
+				.count() > 0);
 
 		assertEquals(338, sw_simrel.count());