added parameter to drive the graph merge strategy: priority (BETA|PROD)

This commit is contained in:
Claudio Atzori 2020-07-20 10:48:01 +02:00
parent 94ccdb4852
commit e0c4cf6f7b
3 changed files with 53 additions and 8 deletions

View File

@ -37,6 +37,8 @@ public class MergeGraphSparkJob {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String PRIORITY_DEFAULT = "BETA"; // BETA | PROD
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
String jsonConfiguration = IOUtils String jsonConfiguration = IOUtils
@ -47,6 +49,11 @@ public class MergeGraphSparkJob {
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
parser.parseArgument(args); parser.parseArgument(args);
String priority = Optional
.ofNullable(parser.get("priority"))
.orElse(PRIORITY_DEFAULT);
log.info("priority: {}", priority);
Boolean isSparkSessionManaged = Optional Boolean isSparkSessionManaged = Optional
.ofNullable(parser.get("isSparkSessionManaged")) .ofNullable(parser.get("isSparkSessionManaged"))
.map(Boolean::valueOf) .map(Boolean::valueOf)
@ -76,12 +83,13 @@ public class MergeGraphSparkJob {
isSparkSessionManaged, isSparkSessionManaged,
spark -> { spark -> {
removeOutputDir(spark, outputPath); removeOutputDir(spark, outputPath);
mergeGraphTable(spark, betaInputPath, prodInputPath, entityClazz, entityClazz, outputPath); mergeGraphTable(spark, priority, betaInputPath, prodInputPath, entityClazz, entityClazz, outputPath);
}); });
} }
private static <P extends Oaf, B extends Oaf> void mergeGraphTable( private static <P extends Oaf, B extends Oaf> void mergeGraphTable(
SparkSession spark, SparkSession spark,
String priority,
String betaInputPath, String betaInputPath,
String prodInputPath, String prodInputPath,
Class<P> p_clazz, Class<P> p_clazz,
@ -96,6 +104,22 @@ public class MergeGraphSparkJob {
.map((MapFunction<Tuple2<Tuple2<String, P>, Tuple2<String, B>>, P>) value -> { .map((MapFunction<Tuple2<Tuple2<String, P>, Tuple2<String, B>>, P>) value -> {
Optional<P> p = Optional.ofNullable(value._1()).map(Tuple2::_2); Optional<P> p = Optional.ofNullable(value._1()).map(Tuple2::_2);
Optional<B> b = Optional.ofNullable(value._2()).map(Tuple2::_2); Optional<B> b = Optional.ofNullable(value._2()).map(Tuple2::_2);
switch (priority) {
default:
case "BETA":
return mergeWithPriorityToBETA(p, b);
case "PROD":
return mergeWithPriorityToPROD(p, b);
}
}, Encoders.bean(p_clazz))
.filter((FilterFunction<P>) Objects::nonNull)
.write()
.mode(SaveMode.Overwrite)
.option("compression", "gzip")
.json(outputPath);
}
private static <P extends Oaf, B extends Oaf> P mergeWithPriorityToPROD(Optional<P> p, Optional<B> b) {
if (b.isPresent() & !p.isPresent()) { if (b.isPresent() & !p.isPresent()) {
return (P) b.get(); return (P) b.get();
} }
@ -103,12 +127,16 @@ public class MergeGraphSparkJob {
return p.get(); return p.get();
} }
return null; return null;
}, Encoders.bean(p_clazz)) }
.filter((FilterFunction<P>) Objects::nonNull)
.write() private static <P extends Oaf, B extends Oaf> P mergeWithPriorityToBETA(Optional<P> p, Optional<B> b) {
.mode(SaveMode.Overwrite) if (p.isPresent() & !b.isPresent()) {
.option("compression", "gzip") return p.get();
.json(outputPath); }
if (b.isPresent()) {
return (P) b.get();
}
return null;
} }
private static <T extends Oaf> Dataset<Tuple2<String, T>> readTableFromPath( private static <T extends Oaf> Dataset<Tuple2<String, T>> readTableFromPath(

View File

@ -13,6 +13,10 @@
<name>graphOutputPath</name> <name>graphOutputPath</name>
<description>the output merged graph root path</description> <description>the output merged graph root path</description>
</property> </property>
<property>
<name>priority</name>
<description>decides from which infrastructure the content must win in case of ID clash</description>
</property>
<property> <property>
<name>sparkDriverMemory</name> <name>sparkDriverMemory</name>
@ -88,6 +92,7 @@
<arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/publication</arg> <arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/publication</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/publication</arg> <arg>--outputPath</arg><arg>${graphOutputPath}/publication</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg> <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
<arg>--priority</arg><arg>${priority}</arg>
</spark> </spark>
<ok to="wait_merge"/> <ok to="wait_merge"/>
<error to="Kill"/> <error to="Kill"/>
@ -114,6 +119,7 @@
<arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/dataset</arg> <arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/dataset</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/dataset</arg> <arg>--outputPath</arg><arg>${graphOutputPath}/dataset</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg> <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
<arg>--priority</arg><arg>${priority}</arg>
</spark> </spark>
<ok to="wait_merge"/> <ok to="wait_merge"/>
<error to="Kill"/> <error to="Kill"/>
@ -140,6 +146,7 @@
<arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/otherresearchproduct</arg> <arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/otherresearchproduct</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/otherresearchproduct</arg> <arg>--outputPath</arg><arg>${graphOutputPath}/otherresearchproduct</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg> <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
<arg>--priority</arg><arg>${priority}</arg>
</spark> </spark>
<ok to="wait_merge"/> <ok to="wait_merge"/>
<error to="Kill"/> <error to="Kill"/>
@ -166,6 +173,7 @@
<arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/software</arg> <arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/software</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/software</arg> <arg>--outputPath</arg><arg>${graphOutputPath}/software</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg> <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
<arg>--priority</arg><arg>${priority}</arg>
</spark> </spark>
<ok to="wait_merge"/> <ok to="wait_merge"/>
<error to="Kill"/> <error to="Kill"/>
@ -192,6 +200,7 @@
<arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/datasource</arg> <arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/datasource</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/datasource</arg> <arg>--outputPath</arg><arg>${graphOutputPath}/datasource</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Datasource</arg> <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Datasource</arg>
<arg>--priority</arg><arg>${priority}</arg>
</spark> </spark>
<ok to="wait_merge"/> <ok to="wait_merge"/>
<error to="Kill"/> <error to="Kill"/>
@ -218,6 +227,7 @@
<arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/organization</arg> <arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/organization</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/organization</arg> <arg>--outputPath</arg><arg>${graphOutputPath}/organization</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Organization</arg> <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Organization</arg>
<arg>--priority</arg><arg>${priority}</arg>
</spark> </spark>
<ok to="wait_merge"/> <ok to="wait_merge"/>
<error to="Kill"/> <error to="Kill"/>
@ -244,6 +254,7 @@
<arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/project</arg> <arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/project</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/project</arg> <arg>--outputPath</arg><arg>${graphOutputPath}/project</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Project</arg> <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Project</arg>
<arg>--priority</arg><arg>${priority}</arg>
</spark> </spark>
<ok to="wait_merge"/> <ok to="wait_merge"/>
<error to="Kill"/> <error to="Kill"/>
@ -266,11 +277,11 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.shuffle.partitions=7680 --conf spark.sql.shuffle.partitions=7680
</spark-opts> </spark-opts>
<arg>--betaInputPath</arg><arg>${betaInputGgraphPath}/relation</arg> <arg>--betaInputPath</arg><arg>${betaInputGgraphPath}/relation</arg>
<arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/relation</arg> <arg>--prodInputPath</arg><arg>${prodInputGgraphPath}/relation</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/relation</arg> <arg>--outputPath</arg><arg>${graphOutputPath}/relation</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Relation</arg> <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Relation</arg>
<arg>--priority</arg><arg>${priority}</arg>
</spark> </spark>
<ok to="wait_merge"/> <ok to="wait_merge"/>
<error to="Kill"/> <error to="Kill"/>

View File

@ -28,5 +28,11 @@
"paramLongName": "graphTableClassName", "paramLongName": "graphTableClassName",
"paramDescription": "class name moelling the graph table", "paramDescription": "class name moelling the graph table",
"paramRequired": true "paramRequired": true
},
{
"paramName": "pr",
"paramLongName": "priority",
"paramDescription": "decides from which infrastructure the content must win in case of ID clash",
"paramRequired": false
} }
] ]