package eu.dnetlib.dhp.oa.graph.dump.country; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.io.Serializable; import java.util.*; import javax.rmi.CORBA.Util; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.graph.dump.Constants; import eu.dnetlib.dhp.oa.graph.dump.ResultMapper; import eu.dnetlib.dhp.oa.graph.dump.Utils; import eu.dnetlib.dhp.oa.graph.dump.subset.MasterDuplicate; import eu.dnetlib.dhp.oa.graph.dump.subset.SparkDumpResult; import eu.dnetlib.dhp.oa.graph.dump.subset.criteria.VerbResolver; import eu.dnetlib.dhp.oa.graph.dump.subset.criteria.VerbResolverFactory; import eu.dnetlib.dhp.oa.graph.dump.subset.selectionconstraints.Param; import eu.dnetlib.dhp.oa.graph.dump.subset.selectionconstraints.SelectionConstraints; import eu.dnetlib.dhp.oa.model.graph.GraphResult; import eu.dnetlib.dhp.schema.oaf.DataInfo; import eu.dnetlib.dhp.schema.oaf.KeyValue; import scala.Tuple2; /** * @author miriam.baglioni * @Date 27/04/23 * Selects the results having in the country the given country */ public class SparkFindResultWithCountry implements Serializable { private static final Logger log = LoggerFactory.getLogger(SparkFindResultWithCountry.class); public static final String COMPRESSION = "compression"; public static final String GZIP = "gzip"; public static void main(String[] args) throws Exception { String jsonConfiguration = IOUtils .toString( SparkFindResultWithCountry.class .getResourceAsStream( "/eu/dnetlib/dhp/oa/graph/dump/result_country_parameters.json")); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); parser.parseArgument(args); Boolean isSparkSessionManaged = Optional .ofNullable(parser.get("isSparkSessionManaged")) .map(Boolean::valueOf) .orElse(Boolean.TRUE); log.info("isSparkSessionManaged: {}", isSparkSessionManaged); final String inputPath = parser.get("sourcePath"); log.info("inputPath: {}", inputPath); final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); final String resultType = parser.get("resultType"); log.info("resultType: {}", resultType); final String resultClassName = parser.get("resultTableName"); log.info("resultTableName: {}", resultClassName); final String preparedInfoPath = parser.get("resultWithCountry"); Class inputClazz = (Class) Class .forName(resultClassName); run( isSparkSessionManaged, inputPath, outputPath, inputClazz, resultType, preparedInfoPath); } private static void run(Boolean isSparkSessionManaged, String inputPath, String outputPath, Class inputClazz, String resultType, String preparedInfoPath) { SparkConf conf = new SparkConf(); runWithSparkSession( conf, isSparkSessionManaged, spark -> { Utils.removeOutputDir(spark, outputPath + "/original/" + resultType); resultDump( spark, inputPath, outputPath, inputClazz, resultType, preparedInfoPath); }); } public static void resultDump( SparkSession spark, String inputPath, String outputPath, Class inputClazz, String resultType, String preparedInfoPath) { Dataset resultsWithCountry = spark.read().textFile(preparedInfoPath).distinct(); Dataset result = Utils .readPath(spark, inputPath, inputClazz) .filter( (FilterFunction) r -> !r.getDataInfo().getInvisible() && !r.getDataInfo().getDeletedbyinference()); resultsWithCountry .joinWith(result, resultsWithCountry.col("value").equalTo(result.col("id"))) .map((MapFunction, I>) t2 -> t2._2(), Encoders.bean(inputClazz)) .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") .json(outputPath + "/original/" + resultType); } }