|
|
|
@ -1,17 +1,15 @@
|
|
|
|
|
|
|
|
|
|
package eu.dnetlib.dhp.oa.merge;
|
|
|
|
|
|
|
|
|
|
import com.fasterxml.jackson.databind.DeserializationFeature;
|
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
|
import com.jayway.jsonpath.Configuration;
|
|
|
|
|
import com.jayway.jsonpath.DocumentContext;
|
|
|
|
|
import com.jayway.jsonpath.JsonPath;
|
|
|
|
|
import com.jayway.jsonpath.Option;
|
|
|
|
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
|
|
|
|
import eu.dnetlib.dhp.common.HdfsSupport;
|
|
|
|
|
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.*;
|
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
|
|
|
|
|
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
|
|
|
|
import static eu.dnetlib.dhp.utils.DHPUtils.toSeq;
|
|
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.Objects;
|
|
|
|
|
import java.util.Optional;
|
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
|
|
|
|
import org.apache.commons.io.IOUtils;
|
|
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
|
|
import org.apache.spark.SparkConf;
|
|
|
|
@ -22,16 +20,20 @@ import org.apache.spark.sql.*;
|
|
|
|
|
import org.apache.spark.sql.expressions.Aggregator;
|
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
import scala.Tuple2;
|
|
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.Objects;
|
|
|
|
|
import java.util.Optional;
|
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
import com.fasterxml.jackson.databind.DeserializationFeature;
|
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
|
import com.jayway.jsonpath.Configuration;
|
|
|
|
|
import com.jayway.jsonpath.DocumentContext;
|
|
|
|
|
import com.jayway.jsonpath.JsonPath;
|
|
|
|
|
import com.jayway.jsonpath.Option;
|
|
|
|
|
|
|
|
|
|
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
|
|
|
|
import static eu.dnetlib.dhp.utils.DHPUtils.toSeq;
|
|
|
|
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
|
|
|
|
import eu.dnetlib.dhp.common.HdfsSupport;
|
|
|
|
|
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.*;
|
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
|
|
|
|
|
import scala.Tuple2;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Groups the graph content by entity identifier to ensure ID uniqueness
|
|
|
|
|