refactoring

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/BipDeserialize.java

@@ -1,3 +1,4 @@
+
 package eu.dnetlib.dhp.actionmanager.bipfinder;
 
 import java.io.Serializable;
@@ -5,7 +6,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 
-
 public class BipDeserialize extends HashMap<String, List<Score>> implements Serializable {
 
 	public BipDeserialize() {

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/BipScore.java

@@ -1,3 +1,4 @@
+
 package eu.dnetlib.dhp.actionmanager.bipfinder;
 
 import java.io.Serializable;

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/KeyValue.java

@@ -1,3 +1,4 @@
+
 package eu.dnetlib.dhp.actionmanager.bipfinder;
 
 import java.io.Serializable;

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/PreparedResult.java

@@ -1,3 +1,4 @@
+
 package eu.dnetlib.dhp.actionmanager.bipfinder;
 
 import java.io.Serializable;

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/Score.java

@@ -1,3 +1,4 @@
+
 package eu.dnetlib.dhp.actionmanager.bipfinder;
 
 import java.io.Serializable;

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java

@@ -1,11 +1,13 @@
+
 package eu.dnetlib.dhp.actionmanager.bipfinder;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
-import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+
-import eu.dnetlib.dhp.common.HdfsSupport;
+import java.io.Serializable;
-import eu.dnetlib.dhp.schema.action.AtomicAction;
+import java.util.List;
-import eu.dnetlib.dhp.schema.oaf.*;
+import java.util.Optional;
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import java.util.stream.Collectors;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos;
 import org.apache.hadoop.io.Text;
@@ -22,15 +24,16 @@ import org.apache.spark.sql.Row;
 import org.apache.spark.sql.SparkSession;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.HdfsSupport;
+import eu.dnetlib.dhp.schema.action.AtomicAction;
+import eu.dnetlib.dhp.schema.oaf.*;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
 import scala.Tuple2;
 
-import java.io.Serializable;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
-
 public class SparkAtomicActionScoreJob implements Serializable {
 
 	private static String DOI = "doi";
@@ -86,18 +89,17 @@ public class SparkAtomicActionScoreJob implements Serializable {
 
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
-        JavaRDD<BipDeserialize> bipDeserializeJavaRDD = sc.textFile(bipScorePath)
+		JavaRDD<BipDeserialize> bipDeserializeJavaRDD = sc
+			.textFile(bipScorePath)
 			.map(item -> OBJECT_MAPPER.readValue(item, BipDeserialize.class));
 
-
+		Dataset<BipScore> bipScores = spark
-        Dataset<BipScore> bipScores = spark.createDataset(bipDeserializeJavaRDD.flatMap(entry ->
+			.createDataset(bipDeserializeJavaRDD.flatMap(entry -> entry.keySet().stream().map(key -> {
-                entry.keySet().stream().map(key -> {
 				BipScore bs = new BipScore();
 				bs.setId(key);
 				bs.setScoreList(entry.get(key));
 				return bs;
-                }).collect(Collectors.toList()).iterator()
+			}).collect(Collectors.toList()).iterator()).rdd(), Encoders.bean(BipScore.class));
-        ).rdd(), Encoders.bean(BipScore.class));
 
 		System.out.println(bipScores.count());
 
@@ -105,13 +107,17 @@ public class SparkAtomicActionScoreJob implements Serializable {
 
 		results.createOrReplaceTempView("result");
 
-        Dataset<PreparedResult> preparedResult = spark.sql("select pIde.value value, id " +
+		Dataset<PreparedResult> preparedResult = spark
+			.sql(
+				"select pIde.value value, id " +
 					"from result " +
 					"lateral view explode (pid) p as pIde " +
 					"where dataInfo.deletedbyinference = false and pIde.qualifier.classid = '" + DOI + "'")
 			.as(Encoders.bean(PreparedResult.class));
 
-        Dataset<BipScore> tmp = bipScores.joinWith(preparedResult, bipScores.col("id").equalTo(preparedResult.col("value")),
+		Dataset<BipScore> tmp = bipScores
+			.joinWith(
+				preparedResult, bipScores.col("id").equalTo(preparedResult.col("value")),
 				"inner")
 			.map((MapFunction<Tuple2<BipScore, PreparedResult>, BipScore>) value -> {
 				BipScore ret = value._1();
@@ -119,9 +125,9 @@ public class SparkAtomicActionScoreJob implements Serializable {
 				return ret;
 			}, Encoders.bean(BipScore.class));
 
-        tmp.groupByKey((MapFunction<BipScore, String>) value -> value.getId(), Encoders.STRING())
+		tmp
-        .mapGroups((MapGroupsFunction<String, BipScore, I>) (k, it) ->
+			.groupByKey((MapFunction<BipScore, String>) value -> value.getId(), Encoders.STRING())
-        {
+			.mapGroups((MapGroupsFunction<String, BipScore, I>) (k, it) -> {
 				Result ret = inputClazz.newInstance();
 				BipScore first = it.next();
 				ret.setId(first.getId());
@@ -141,36 +147,40 @@ public class SparkAtomicActionScoreJob implements Serializable {
 	}
 
 	public static Dataset<BipScore> getBipScoreDataset(Dataset<BipDeserialize> bipdeserialized) {
-        return  bipdeserialized.flatMap((FlatMapFunction<BipDeserialize, BipScore>) bip ->
+		return bipdeserialized
-                        bip.keySet().stream().map(key -> {
+			.flatMap((FlatMapFunction<BipDeserialize, BipScore>) bip -> bip.keySet().stream().map(key -> {
 				BipScore bs = new BipScore();
 				bs.setId(key);
 				bs.setScoreList(bip.get(key));
 				return bs;
-                        }).collect(Collectors.toList()).iterator()
+			}).collect(Collectors.toList()).iterator(), Encoders.bean(BipScore.class));
-                , Encoders.bean(BipScore.class));
 	}
 
 	private static List<Measure> getMeasure(BipScore value) {
-        return value.getScoreList()
+		return value
+			.getScoreList()
 			.stream()
 			.map(score -> {
 				Measure m = new Measure();
 				m.setId(score.getId());
-                    m.setUnit(score.getUnit().stream()
+				m
+					.setUnit(
+						score
+							.getUnit()
+							.stream()
 							.map(unit -> {
 								KeyValue kv = new KeyValue();
 								kv.setValue(unit.getValue());
 								kv.setKey(unit.getKey());
 								kv.setDataInfo(getDataInfo());
 								return kv;
-                            }).collect(Collectors.toList()));
+							})
+							.collect(Collectors.toList()));
 				return m;
-                }).collect(Collectors.toList());
+			})
+			.collect(Collectors.toList());
 	}
 
-
-
 	private static DataInfo getDataInfo() {
 		DataInfo di = new DataInfo();
 		di.setInferred(false);
@@ -186,7 +196,6 @@ public class SparkAtomicActionScoreJob implements Serializable {
 		return di;
 	}
 
-
 	private static void removeOutputDir(SparkSession spark, String path) {
 		HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration());
 	}
@@ -199,5 +208,4 @@ public class SparkAtomicActionScoreJob implements Serializable {
 			.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
 	}
 
-
 }

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJobTest.java

@@ -1,9 +1,12 @@
+
 package eu.dnetlib.dhp.actionmanager.bipfinder;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
-import eu.dnetlib.dhp.schema.action.AtomicAction;
+import java.nio.file.Files;
-import eu.dnetlib.dhp.schema.oaf.*;
+import java.nio.file.Path;
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import java.util.List;
+import java.util.stream.Collectors;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
@@ -24,13 +27,13 @@ import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import scala.Tuple2;
 
-import java.io.IOException;
+import com.fasterxml.jackson.databind.ObjectMapper;
-import java.nio.file.Files;
+
-import java.nio.file.Path;
+import eu.dnetlib.dhp.schema.action.AtomicAction;
-import java.util.List;
+import eu.dnetlib.dhp.schema.oaf.*;
-import java.util.stream.Collectors;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import scala.Tuple2;
 
 public class SparkAtomicActionScoreJobTest {
 
@@ -73,13 +76,16 @@ public class SparkAtomicActionScoreJobTest {
 
 	@Test
 	public void matchOne() throws Exception {
-        String bipScoresPath =  getClass().getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json").getPath();
+		String bipScoresPath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json")
+			.getPath();
 		String inputPath = getClass()
 			.getResource(
 				"/eu/dnetlib/dhp/actionmanager/bipfinder/publication.json")
 			.getPath();
 
-        SparkAtomicActionScoreJob.main(
+		SparkAtomicActionScoreJob
+			.main(
 				new String[] {
 					"-isSparkSessionManaged",
 					Boolean.FALSE.toString(),
@@ -93,7 +99,6 @@ public class SparkAtomicActionScoreJobTest {
 					workingDir.toString() + "/actionSet"
 				});
 
-
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Publication> tmp = sc
@@ -106,32 +111,43 @@ public class SparkAtomicActionScoreJobTest {
 		Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class));
 		verificationDataset.createOrReplaceTempView("publication");
 
-        Dataset<Row> execVerification = spark.sql("Select p.id oaid, mes.id, mUnit.value from publication p " +
+		Dataset<Row> execVerification = spark
+			.sql(
+				"Select p.id oaid, mes.id, mUnit.value from publication p " +
 					"lateral view explode(measures) m as mes " +
 					"lateral view explode(mes.unit) u as mUnit ");
 
 		Assertions.assertEquals(2, execVerification.count());
 
-        Assertions.assertEquals("50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb",
+		Assertions
+			.assertEquals(
+				"50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb",
 				execVerification.select("oaid").collectAsList().get(0).getString(0));
 
-        Assertions.assertEquals("1.47565045883e-08",
+		Assertions
+			.assertEquals(
+				"1.47565045883e-08",
 				execVerification.filter("id = 'influence'").select("value").collectAsList().get(0).getString(0));
 
-        Assertions.assertEquals("0.227515392",
+		Assertions
+			.assertEquals(
+				"0.227515392",
 				execVerification.filter("id = 'popularity'").select("value").collectAsList().get(0).getString(0));
 
 	}
 
 	@Test
 	public void matchOneWithTwo() throws Exception {
-        String bipScoresPath =  getClass().getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json").getPath();
+		String bipScoresPath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json")
+			.getPath();
 		String inputPath = getClass()
 			.getResource(
 				"/eu/dnetlib/dhp/actionmanager/bipfinder/publication_2.json")
 			.getPath();
 
-        SparkAtomicActionScoreJob.main(
+		SparkAtomicActionScoreJob
+			.main(
 				new String[] {
 					"-isSparkSessionManaged",
 					Boolean.FALSE.toString(),
@@ -145,7 +161,6 @@ public class SparkAtomicActionScoreJobTest {
 					workingDir.toString() + "/actionSet"
 				});
 
-
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Publication> tmp = sc
@@ -158,45 +173,58 @@ public class SparkAtomicActionScoreJobTest {
 		Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class));
 		verificationDataset.createOrReplaceTempView("publication");
 
-        Dataset<Row> execVerification = spark.sql("Select p.id oaid, mes.id, mUnit.value from publication p " +
+		Dataset<Row> execVerification = spark
+			.sql(
+				"Select p.id oaid, mes.id, mUnit.value from publication p " +
 					"lateral view explode(measures) m as mes " +
 					"lateral view explode(mes.unit) u as mUnit ");
 
 		Assertions.assertEquals(4, execVerification.count());
 
-
+		Assertions
-        Assertions.assertEquals("50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb",
+			.assertEquals(
+				"50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb",
 				execVerification.select("oaid").collectAsList().get(0).getString(0));
 
-        Assertions.assertEquals(2,
+		Assertions
+			.assertEquals(
+				2,
 				execVerification.filter("id = 'influence'").count());
 
-        Assertions.assertEquals(2,
+		Assertions
+			.assertEquals(
+				2,
 				execVerification.filter("id = 'popularity'").count());
 
 		List<Row> tmp_ds = execVerification.filter("id = 'influence'").select("value").collectAsList();
 		String tmp_influence = tmp_ds.get(0).getString(0);
-        Assertions.assertTrue("1.47565045883e-08".equals(tmp_influence)||
+		Assertions
+			.assertTrue(
+				"1.47565045883e-08".equals(tmp_influence) ||
 					"1.98956540239e-08".equals(tmp_influence));
 
 		tmp_influence = tmp_ds.get(1).getString(0);
-        Assertions.assertTrue("1.47565045883e-08".equals(tmp_influence)||
+		Assertions
+			.assertTrue(
+				"1.47565045883e-08".equals(tmp_influence) ||
 					"1.98956540239e-08".equals(tmp_influence));
 
 		Assertions.assertTrue(!tmp_ds.get(0).getString(0).equals(tmp_ds.get(1).getString(0)));
 
-
 	}
 
 	@Test
 	public void matchTwo() throws Exception {
-        String bipScoresPath =  getClass().getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json").getPath();
+		String bipScoresPath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/bipfinder/bip_scores.json")
+			.getPath();
 		String inputPath = getClass()
 			.getResource(
 				"/eu/dnetlib/dhp/actionmanager/bipfinder/publication_3.json")
 			.getPath();
 
-        SparkAtomicActionScoreJob.main(
+		SparkAtomicActionScoreJob
+			.main(
 				new String[] {
 					"-isSparkSessionManaged",
 					Boolean.FALSE.toString(),
@@ -210,7 +238,6 @@ public class SparkAtomicActionScoreJobTest {
 					workingDir.toString() + "/actionSet"
 				});
 
-
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Publication> tmp = sc
@@ -223,41 +250,81 @@ public class SparkAtomicActionScoreJobTest {
 		Dataset<Publication> verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(Publication.class));
 		verificationDataset.createOrReplaceTempView("publication");
 
-        Dataset<Row> execVerification = spark.sql("Select p.id oaid, mes.id, mUnit.value from publication p " +
+		Dataset<Row> execVerification = spark
+			.sql(
+				"Select p.id oaid, mes.id, mUnit.value from publication p " +
 					"lateral view explode(measures) m as mes " +
 					"lateral view explode(mes.unit) u as mUnit ");
 
 		Assertions.assertEquals(4, execVerification.count());
 
-
+		Assertions
-        Assertions.assertEquals(2,
+			.assertEquals(
+				2,
 				execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb'").count());
 
-        Assertions.assertEquals(2,
+		Assertions
+			.assertEquals(
+				2,
 				execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09'").count());
 
-
+		Assertions
-        Assertions.assertEquals(2,
+			.assertEquals(
+				2,
 				execVerification.filter("id = 'influence'").count());
 
-        Assertions.assertEquals(2,
+		Assertions
+			.assertEquals(
+				2,
 				execVerification.filter("id = 'popularity'").count());
 
-        Assertions.assertEquals("1.47565045883e-08" ,
+		Assertions
-                execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " +
+			.assertEquals(
-                        "and id = 'influence'").select("value").collectAsList().get(0).getString(0));
+				"1.47565045883e-08",
+				execVerification
+					.filter(
+						"oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " +
+							"and id = 'influence'")
+					.select("value")
+					.collectAsList()
+					.get(0)
+					.getString(0));
 
-        Assertions.assertEquals(
+		Assertions
-                "1.98956540239e-08", execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " +
+			.assertEquals(
-                        "and id = 'influence'").select("value").collectAsList().get(0).getString(0));
+				"1.98956540239e-08",
+				execVerification
+					.filter(
+						"oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " +
+							"and id = 'influence'")
+					.select("value")
+					.collectAsList()
+					.get(0)
+					.getString(0));
 
-        Assertions.assertEquals(
+		Assertions
-                "0.282046161584", execVerification.filter("oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " +
+			.assertEquals(
-                        "and id = 'popularity'").select("value").collectAsList().get(0).getString(0));
+				"0.282046161584",
+				execVerification
+					.filter(
+						"oaid = '50|acm_________::faed5b7a1bd8f51118d13ed29cfaee09' " +
+							"and id = 'popularity'")
+					.select("value")
+					.collectAsList()
+					.get(0)
+					.getString(0));
 
-        Assertions.assertEquals("0.227515392" ,
+		Assertions
-                execVerification.filter("oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " +
+			.assertEquals(
-                        "and id = 'popularity'").select("value").collectAsList().get(0).getString(0));
+				"0.227515392",
+				execVerification
+					.filter(
+						"oaid = '50|355e65625b88::ffa5bad14f4adc0c9a15c00efbbccddb' " +
+							"and id = 'popularity'")
+					.select("value")
+					.collectAsList()
+					.get(0)
+					.getString(0));
 
 	}