set configuration property in openorgs duplicates wf

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java

@@ -1,3 +1,4 @@
+
 package eu.dnetlib.dhp.oa.dedup;
 
 import java.io.IOException;
@@ -5,10 +6,7 @@ import java.util.Optional;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.FilterFunction;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.api.java.function.PairFunction;
 import org.apache.spark.sql.Dataset;
@@ -21,7 +19,6 @@ import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXException;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
-import eu.dnetlib.dhp.oa.dedup.model.Block;
 import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.Relation;
 import eu.dnetlib.dhp.utils.ISLookupClientFactory;
@@ -31,7 +28,6 @@ import eu.dnetlib.pace.config.DedupConfig;
 import eu.dnetlib.pace.model.MapDocument;
 import eu.dnetlib.pace.util.MapDocumentUtil;
 import scala.Tuple2;
-import scala.Tuple3;
 
 public class SparkWhitelistSimRels extends AbstractSparkAction {
 
@@ -81,10 +77,12 @@ public class SparkWhitelistSimRels extends AbstractSparkAction {
 
 		JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
 
-        //file format: source####target
+		// file format: source####target
-        Dataset<Tuple2<String, String>> whiteListRels = spark.createDataset(sc
+		Dataset<Tuple2<String, String>> whiteListRels = spark
+			.createDataset(
+				sc
 					.textFile(whiteListPath)
-                        //check if the line is in the correct format: id1####id2
+					// check if the line is in the correct format: id1####id2
 					.filter(s -> s.contains(WHITELIST_SEPARATOR) && s.split(WHITELIST_SEPARATOR).length == 2)
 					.map(s -> new Tuple2<>(s.split(WHITELIST_SEPARATOR)[0], s.split(WHITELIST_SEPARATOR)[1]))
 					.rdd(),
@@ -99,7 +97,9 @@ public class SparkWhitelistSimRels extends AbstractSparkAction {
 
 			final String outputPath = DedupUtility.createSimRelPath(workingPath, actionSetId, subEntity);
 
-            Dataset<Tuple2<String, String>> entities = spark.createDataset(sc
+			Dataset<Tuple2<String, String>> entities = spark
+				.createDataset(
+					sc
 						.textFile(DedupUtility.createEntityPath(graphBasePath, subEntity))
 						.repartition(numPartitions)
 						.mapToPair(
@@ -112,17 +112,20 @@ public class SparkWhitelistSimRels extends AbstractSparkAction {
 
 			Dataset<Tuple2<String, String>> whiteListRels1 = whiteListRels
 				.joinWith(entities, whiteListRels.col("_1").equalTo(entities.col("_1")), "inner")
-                    .map((MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1, Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+				.map(
+					(MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1,
+					Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
 
 			Dataset<Tuple2<String, String>> whiteListRels2 = whiteListRels1
 				.joinWith(entities, whiteListRels1.col("_2").equalTo(entities.col("_1")), "inner")
-                    .map((MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1, Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+				.map(
+					(MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1,
+					Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
 
 			Dataset<Relation> whiteListSimRels = whiteListRels2
-                    .map((MapFunction<Tuple2<String, String>, Relation>)
+				.map(
-                                    r -> createSimRel(r._1(), r._2(), entity),
+					(MapFunction<Tuple2<String, String>, Relation>) r -> createSimRel(r._1(), r._2(), entity),
-                                    Encoders.bean(Relation.class)
+					Encoders.bean(Relation.class));
-                    );
 
 			saveParquet(whiteListSimRels, outputPath, SaveMode.Append);
 		}

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java

@@ -1,7 +1,8 @@
+
 package eu.dnetlib.dhp.oa.dedup;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.concurrent.TimeUnit;
-import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
@@ -10,7 +11,9 @@ import org.apache.http.impl.client.HttpClients;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 
 public class UpdateOpenorgsJob {
 
@@ -18,7 +21,10 @@ public class UpdateOpenorgsJob {
 
 	public static void main(String[] args) throws Exception {
 		ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                    IOUtils.toString(SparkCreateSimRels.class.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json")));
+			IOUtils
+				.toString(
+					SparkCreateSimRels.class
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json")));
 		parser.parseArgument(args);
 
 		final String apiUrl = parser.get("apiUrl");
@@ -28,17 +34,17 @@ public class UpdateOpenorgsJob {
 		log.info("delay: '{}'", delay);
 
 		APIResponse res = httpCall(apiUrl);
-		while(res!=null && res.getStatus().equals(ImportStatus.RUNNING)){
+		while (res != null && res.getStatus().equals(ImportStatus.RUNNING)) {
 			TimeUnit.MINUTES.sleep(delay);
 			res = httpCall(apiUrl + "/status");
 		}
 
-		if (res==null) {
+		if (res == null) {
 			log.error("Openorgs Update FAILED: No response");
 			throw new RuntimeException("Openorgs Update FAILED: No response");
 		}
 
-		if (res.getStatus()==null || !res.getStatus().equals(ImportStatus.SUCCESS)) {
+		if (res.getStatus() == null || !res.getStatus().equals(ImportStatus.SUCCESS)) {
 			log.error("Openorgs Update FAILED: '{}' - '{}'", res.getStatus(), res.getMessage());
 			throw new RuntimeException(res.getMessage());
 		}
@@ -107,9 +113,5 @@ class APIResponse {
 }
 
 enum ImportStatus {
-	SUCCESS,
+	SUCCESS, FAILED, RUNNING, NOT_LAUNCHED, NOT_YET_STARTED
-	FAILED,
-	RUNNING,
-	NOT_LAUNCHED,
-	NOT_YET_STARTED
 }

dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml

@@ -267,6 +267,12 @@
 
     <action name="update_openorgs">
         <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
             <main-class>eu.dnetlib.dhp.oa.dedup.UpdateOpenorgsJob</main-class>
             <arg>--apiUrl</arg><arg>${apiUrl}</arg>
             <arg>--delay</arg><arg>5</arg>

dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java

@@ -171,7 +171,7 @@ public class SparkDedupTest implements Serializable {
 
 		parser
 			.parseArgument(
-                        new String[]{
+				new String[] {
 					"-i", testGraphBasePath,
 					"-asi", testActionSetId,
 					"-la", "lookupurl",
@@ -226,7 +226,7 @@ public class SparkDedupTest implements Serializable {
 
 		parser
 			.parseArgument(
-                        new String[]{
+				new String[] {
 					"-i", testGraphBasePath,
 					"-asi", testActionSetId,
 					"-la", "lookupurl",
@@ -257,29 +257,35 @@ public class SparkDedupTest implements Serializable {
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "otherresearchproduct"))
 			.count();
 
-        //entities simrels supposed to be equal to the number of previous step (no rels in whitelist)
+		// entities simrels supposed to be equal to the number of previous step (no rels in whitelist)
 		assertEquals(3082, orgs_simrel);
 		assertEquals(7036, pubs_simrel);
 		assertEquals(442, ds_simrel);
 		assertEquals(6750, orp_simrel);
 
-        //entities simrels to be different from the number of previous step (new simrels in the whitelist)
+		// entities simrels to be different from the number of previous step (new simrels in the whitelist)
 		Dataset<Row> sw_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "software"));
 
-        //check if the first relation in the whitelist exists
+		// check if the first relation in the whitelist exists
-        assertTrue(sw_simrel
+		assertTrue(
+			sw_simrel
 				.as(Encoders.bean(Relation.class))
 				.toJavaRDD()
-                .filter(rel ->
+				.filter(
-                        rel.getSource().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[0]) && rel.getTarget().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[1])).count() > 0);
+					rel -> rel.getSource().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[0])
-        //check if the second relation in the whitelist exists
+						&& rel.getTarget().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[1]))
-        assertTrue(sw_simrel
+				.count() > 0);
+		// check if the second relation in the whitelist exists
+		assertTrue(
+			sw_simrel
 				.as(Encoders.bean(Relation.class))
 				.toJavaRDD()
-                .filter(rel ->
+				.filter(
-                        rel.getSource().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[0]) && rel.getTarget().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[1])).count() > 0);
+					rel -> rel.getSource().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[0])
+						&& rel.getTarget().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[1]))
+				.count() > 0);
 
 		assertEquals(338, sw_simrel.count());
 
@@ -298,7 +304,7 @@ public class SparkDedupTest implements Serializable {
 
 		parser
 			.parseArgument(
-                        new String[]{
+				new String[] {
 					"-i",
 					testGraphBasePath,
 					"-asi",
@@ -394,7 +400,7 @@ public class SparkDedupTest implements Serializable {
 
 		parser
 			.parseArgument(
-                        new String[]{
+				new String[] {
 					"-i",
 					testGraphBasePath,
 					"-asi",
@@ -449,7 +455,7 @@ public class SparkDedupTest implements Serializable {
 							"/eu/dnetlib/dhp/oa/dedup/createDedupRecord_parameters.json")));
 		parser
 			.parseArgument(
-                        new String[]{
+				new String[] {
 					"-i",
 					testGraphBasePath,
 					"-asi",
@@ -496,7 +502,7 @@ public class SparkDedupTest implements Serializable {
 							"/eu/dnetlib/dhp/oa/dedup/updateEntity_parameters.json")));
 		parser
 			.parseArgument(
-                        new String[]{
+				new String[] {
 					"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
 				});
 
@@ -612,7 +618,7 @@ public class SparkDedupTest implements Serializable {
 							"/eu/dnetlib/dhp/oa/dedup/propagateRelation_parameters.json")));
 		parser
 			.parseArgument(
-                        new String[]{
+				new String[] {
 					"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
 				});