lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

[Measures] addressed comments in the PR

This commit is contained in:
Miriam Baglioni 2022-04-21 12:09:37 +02:00
parent c304657d91
commit b61efd613b
4 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java
View File

@ -392,7 +392,7 @@ public class OafMapperUtils {
return null; return null;
} }
public static KeyValue newKeyValueInstance (String key, String value, DataInfo dataInfo){ public static KeyValue newKeyValueInstance(String key, String value, DataInfo dataInfo) {
KeyValue kv = new KeyValue(); KeyValue kv = new KeyValue();
kv.setDataInfo(dataInfo); kv.setDataInfo(dataInfo);
kv.setKey(key); kv.setKey(key);

9
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/usagestats/SparkAtomicActionUsageJob.java
View File

@ -22,7 +22,6 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.HdfsSupport; import eu.dnetlib.dhp.common.HdfsSupport;
import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelConstants;
@ -31,7 +30,6 @@ import eu.dnetlib.dhp.schema.oaf.Measure;
import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.dhp.schema.oaf.Result;
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
/** /**
* created the Atomic Action for each tipe of results * created the Atomic Action for each tipe of results
*/ */
@ -65,7 +63,7 @@ public class SparkAtomicActionUsageJob implements Serializable {
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();
conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); conf.set("hive.metastore.uris", parser.get("hive_metastore_uris"));
final String dbname = parser.get("statsdb"); final String dbname = parser.get("usagestatsdb");
final String workingPath = parser.get("workingPath"); final String workingPath = parser.get("workingPath");
@ -130,8 +128,9 @@ public class SparkAtomicActionUsageJob implements Serializable {
return Arrays return Arrays
.asList( .asList(
OafMapperUtils.newMeasureInstance("downloads", String.valueOf(downloads), UPDATE_KEY_USAGE_COUNTS, dataInfo), OafMapperUtils
OafMapperUtils.newMeasureInstance("views", String.valueOf(views), UPDATE_KEY_USAGE_COUNTS, dataInfo)); .newMeasureInstance("downloads", String.valueOf(downloads), UPDATE_KEY_USAGE_COUNTS, dataInfo),
OafMapperUtils.newMeasureInstance("views", String.valueOf(views), UPDATE_KEY_USAGE_COUNTS, dataInfo));
} }

2
dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/usagestats/input_actionset_parameter.json
View File

@ -19,7 +19,7 @@
}, },
{ {
"paramName": "sdb", "paramName": "sdb",
"paramLongName": "statsdb", "paramLongName": "usagestatsdb",
"paramDescription": "the name of the db to be used", "paramDescription": "the name of the db to be used",
"paramRequired": true "paramRequired": true
}, },

6
dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/usagestats/oozie_app/workflow.xml
View File

@ -5,8 +5,8 @@
<description>the path where to store the actionset</description> <description>the path where to store the actionset</description>
</property> </property>
<property> <property>
<name>statsdb</name> <name>usagestatsdb</name>
<description>the path where to store the actionset</description> <description>the name of the db to be used</description>
</property> </property>
<property> <property>
<name>sparkDriverMemory</name> <name>sparkDriverMemory</name>
@ -88,7 +88,7 @@
</spark-opts> </spark-opts>
<arg>--hive_metastore_uris</arg><arg>${hiveMetastoreUris}</arg> <arg>--hive_metastore_uris</arg><arg>${hiveMetastoreUris}</arg>
<arg>--outputPath</arg><arg>${outputPath}</arg> <arg>--outputPath</arg><arg>${outputPath}</arg>
<arg>--statsdb</arg><arg>${statsdb}</arg> <arg>--usagestatsdb</arg><arg>${usagestatsdb}</arg>
<arg>--workingPath</arg><arg>${workingDir}/usageDb</arg> <arg>--workingPath</arg><arg>${workingDir}/usageDb</arg>
</spark> </spark>
<ok to="End"/> <ok to="End"/>