From d52b0e1797e0297b67befcc4e0b9ed514297ce6f Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 7 Aug 2020 17:34:40 +0200 Subject: [PATCH] no use of IsLookUp. The query is done once and its result stored on HDFS. The path to the result is given instead of the isLookUpUrl --- .../community/SparkSplitForCommunity.java | 33 ++----------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/SparkSplitForCommunity.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/SparkSplitForCommunity.java index 9c64efabb..d190369f9 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/SparkSplitForCommunity.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/SparkSplitForCommunity.java @@ -1,29 +1,12 @@ package eu.dnetlib.dhp.oa.graph.dump.community; -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; - import java.io.Serializable; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - import org.apache.commons.io.IOUtils; -import org.apache.spark.SparkConf; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import com.google.gson.Gson; - import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.oa.graph.dump.QueryInformationSystem; -import eu.dnetlib.dhp.oa.graph.dump.Utils; -import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult; -import eu.dnetlib.dhp.utils.ISLookupClientFactory; -import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; public class SparkSplitForCommunity implements Serializable { @@ -51,23 +34,11 @@ public class SparkSplitForCommunity implements Serializable { final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); - final String isLookUpUrl = parser.get("isLookUpUrl"); - log.info("isLookUpUrl: {}", isLookUpUrl); + final String communityMapPath = parser.get("communityMapPath"); CommunitySplit split = new CommunitySplit(); + split.run(isSparkSessionManaged, inputPath, outputPath, communityMapPath); - CommunityMap communityMap; - - QueryInformationSystem queryInformationSystem = new QueryInformationSystem(); - queryInformationSystem.setIsLookUp(getIsLookUpService(isLookUpUrl)); - communityMap = queryInformationSystem.getCommunityMap(); - - split.run(isSparkSessionManaged, inputPath, outputPath, communityMap); - - } - - public static ISLookUpService getIsLookUpService(String isLookUpUrl) { - return ISLookupClientFactory.getLookUpService(isLookUpUrl); } }