|
|
|
@ -3,23 +3,14 @@ package eu.dnetlib.dhp.actionmanager.usagestats;
|
|
|
|
|
|
|
|
|
|
import static eu.dnetlib.dhp.actionmanager.Constants.*;
|
|
|
|
|
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession;
|
|
|
|
|
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
|
|
|
|
|
|
|
|
|
import java.io.Serializable;
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
|
import java.util.Arrays;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.Optional;
|
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
|
|
|
|
import org.apache.commons.io.IOUtils;
|
|
|
|
|
import org.apache.cxf.wsdl.service.factory.MethodNameSoapActionServiceConfiguration;
|
|
|
|
|
import org.apache.hadoop.io.Text;
|
|
|
|
|
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
|
|
|
|
|
import org.apache.spark.SparkConf;
|
|
|
|
|
import org.apache.spark.api.java.JavaRDD;
|
|
|
|
|
import org.apache.spark.api.java.JavaSparkContext;
|
|
|
|
|
import org.apache.spark.api.java.function.ForeachFunction;
|
|
|
|
|
import org.apache.spark.api.java.function.MapFunction;
|
|
|
|
|
import org.apache.spark.api.java.function.MapGroupsFunction;
|
|
|
|
|
import org.apache.spark.sql.Dataset;
|
|
|
|
@ -31,19 +22,15 @@ import org.slf4j.LoggerFactory;
|
|
|
|
|
|
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
|
|
|
|
|
|
import eu.dnetlib.dhp.actionmanager.bipmodel.BipDeserialize;
|
|
|
|
|
import eu.dnetlib.dhp.actionmanager.bipmodel.BipScore;
|
|
|
|
|
|
|
|
|
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
|
|
|
|
import eu.dnetlib.dhp.common.HdfsSupport;
|
|
|
|
|
import eu.dnetlib.dhp.schema.action.AtomicAction;
|
|
|
|
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.DataInfo;
|
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.KeyValue;
|
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.Measure;
|
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.Result;
|
|
|
|
|
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
|
|
|
|
|
import lombok.val;
|
|
|
|
|
import scala.Tuple2;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* created the Atomic Action for each tipe of results
|
|
|
|
@ -143,9 +130,8 @@ public class SparkAtomicActionUsageJob implements Serializable {
|
|
|
|
|
|
|
|
|
|
return Arrays
|
|
|
|
|
.asList(
|
|
|
|
|
Measure
|
|
|
|
|
.newInstance("downloads", String.valueOf(downloads), UPDATE_KEY_USAGE_COUNTS, dataInfo),
|
|
|
|
|
Measure.newInstance("views", String.valueOf(views), UPDATE_KEY_USAGE_COUNTS, dataInfo));
|
|
|
|
|
OafMapperUtils.newMeasureInstance("downloads", String.valueOf(downloads), UPDATE_KEY_USAGE_COUNTS, dataInfo),
|
|
|
|
|
OafMapperUtils.newMeasureInstance("views", String.valueOf(views), UPDATE_KEY_USAGE_COUNTS, dataInfo));
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|