dhp-graph-dump/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/SparkDumpCommunityProducts.java

package eu.dnetlib.dhp.oa.graph.dump.community;

import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;

import java.io.Serializable;
import java.util.*;
import java.util.stream.Collectors;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FilterFunction;
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.databind.ObjectMapper;

import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.dump.Constants;
import eu.dnetlib.dhp.oa.graph.dump.ResultMapper;
import eu.dnetlib.dhp.oa.graph.dump.Utils;
import eu.dnetlib.dhp.oa.graph.dump.exceptions.CardinalityTooHighException;
import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
import eu.dnetlib.dhp.oa.model.community.CommunityResult;
import eu.dnetlib.dhp.schema.common.ModelSupport;
import eu.dnetlib.dhp.schema.oaf.Context;
import eu.dnetlib.dhp.schema.oaf.DataInfo;
import eu.dnetlib.dhp.schema.oaf.OafEntity;
import eu.dnetlib.dhp.schema.oaf.Result;

/**
 * Spark action to trigger the dump of results associated to research community - reseach initiative/infrasctructure The
 * actual dump if performed via the class DumpProducts that is used also for the entire graph dump
 */
public class SparkDumpCommunityProducts implements Serializable {

	private static final Logger log = LoggerFactory.getLogger(SparkDumpCommunityProducts.class);

	public static void main(String[] args) throws Exception {
		String jsonConfiguration = IOUtils
			.toString(
				SparkDumpCommunityProducts.class
					.getResourceAsStream(
						"/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json"));

		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
		parser.parseArgument(args);

		Boolean isSparkSessionManaged = Optional
			.ofNullable(parser.get("isSparkSessionManaged"))
			.map(Boolean::valueOf)
			.orElse(Boolean.TRUE);
		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);

		final String inputPath = parser.get("sourcePath");
		log.info("inputPath: {}", inputPath);

		final String outputPath = parser.get("outputPath");
		log.info("outputPath: {}", outputPath);

		final String resultClassName = parser.get("resultTableName");
		log.info("resultTableName: {}", resultClassName);

		String communityMapPath = Optional
			.ofNullable(parser.get("communityMapPath"))
			.orElse(null);

		String dumpType = Optional
			.ofNullable(parser.get("dumpType"))
			.orElse(null);

		Class<? extends Result> inputClazz = (Class<? extends Result>) Class.forName(resultClassName);

		SparkConf conf = new SparkConf();

		runWithSparkSession(
			conf,
			isSparkSessionManaged,
			spark -> {
				Utils.removeOutputDir(spark, outputPath);
				resultDump(
					spark, inputPath, outputPath, communityMapPath, inputClazz, dumpType);
			});

	}

	public static <I extends OafEntity> void resultDump(
		SparkSession spark,
		String inputPath,
		String outputPath,
		String communityMapPath,
		Class<I> inputClazz,
		String dumpType) {

		CommunityMap communityMap = null;
		if (!StringUtils.isEmpty(communityMapPath))
			communityMap = Utils.getCommunityMap(spark, communityMapPath);

		CommunityMap finalCommunityMap = communityMap;
		Utils
			.readPath(spark, inputPath, inputClazz)
			.map(
				(MapFunction<I, CommunityResult>) value -> execMap(value, finalCommunityMap, dumpType),
				Encoders.bean(CommunityResult.class))
			.filter((FilterFunction<CommunityResult>) value -> value != null)
			.map(
				(MapFunction<CommunityResult, String>) r -> new ObjectMapper().writeValueAsString(r), Encoders.STRING())
			.write()
			.mode(SaveMode.Overwrite)
			.option("compression", "gzip")
			.text(outputPath);

	}

	private static <I extends OafEntity, O extends eu.dnetlib.dhp.oa.model.Result> O execMap(I value,
		CommunityMap communityMap, String dumpType) throws NoAvailableEntityTypeException, CardinalityTooHighException {

		Optional<DataInfo> odInfo = Optional.ofNullable(value.getDataInfo());
		if (Boolean.FALSE.equals(odInfo.isPresent())) {
			return null;
		}
		if (Boolean.TRUE.equals(odInfo.get().getDeletedbyinference())
			|| Boolean.TRUE.equals(odInfo.get().getInvisible())) {
			return null;
		}
		if (StringUtils.isEmpty(dumpType)) {
			Set<String> communities = communityMap.keySet();

			Optional<List<Context>> inputContext = Optional
				.ofNullable(((eu.dnetlib.dhp.schema.oaf.Result) value).getContext());
			if (!inputContext.isPresent()) {
				return null;
			}
			List<String> toDumpFor = inputContext.get().stream().map(c -> {
				if (communities.contains(c.getId())) {
					return c.getId();
				}
				if (c.getId().contains("::") && communities.contains(c.getId().substring(0, c.getId().indexOf("::")))) {
					return c.getId().substring(0, c.getId().indexOf("::"));
				}
				return null;
			}).filter(Objects::nonNull).collect(Collectors.toList());
			if (toDumpFor.isEmpty()) {
				return null;
			}
		}
		return (O) ResultMapper.map(value, communityMap, Constants.DUMPTYPE.COMMUNITY.getType());

	}
}