diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json index b19be2f52f..6e42bfa643 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json @@ -1,11 +1,6 @@ [ - { - "paramName":"cmp", - "paramLongName":"communityMapPath", - "paramDescription": "the path to the serialization of the community map", - "paramRequired": false - }, + { "paramName":"is", "paramLongName":"isLookUpUrl", @@ -13,9 +8,9 @@ "paramRequired": true }, { - "paramName":"s", - "paramLongName":"sourcePath", - "paramDescription": "the path of the sequencial file to read", + "paramName":"nn", + "paramLongName":"nameNode", + "paramDescription": "the name node", "paramRequired": true }, { @@ -23,12 +18,6 @@ "paramLongName": "outputPath", "paramDescription": "the path used to store temporary output files", "paramRequired": true - }, - { - "paramName": "ssm", - "paramLongName": "isSparkSessionManaged", - "paramDescription": "true if the spark session is managed, false otherwise", - "paramRequired": false } ] diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json index a1407ae674..0cc8c2b4cb 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json @@ -1,9 +1,14 @@ [ - { "paramName":"is", "paramLongName":"isLookUpUrl", "paramDescription": "URL of the isLookUp Service", + "paramRequired": true + }, + { + "paramName":"cmp", + "paramLongName":"communityMapPath", + "paramDescription": "the path to the serialization of the community map", "paramRequired": false }, { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml index 167c920b4c..fc3debb64d 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml @@ -81,7 +81,7 @@ - + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] @@ -92,10 +92,20 @@ - + + + + eu.dnetlib.dhp.oa.graph.dump.community.SaveCommunityMap + --outputPath${workingDir}/communityMap + --nameNode${nameNode} + --isLookUpUrl${isLookUpUrl} + + + + @@ -124,6 +134,7 @@ --sourcePath${sourcePath}/publication --resultTableNameeu.dnetlib.dhp.schema.oaf.Publication --outputPath${workingDir}/publication + --communityMapPath${workingDir}/communityMap --isLookUpUrl${isLookUpUrl} @@ -150,6 +161,7 @@ --sourcePath${sourcePath}/dataset --resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset --outputPath${workingDir}/dataset + --communityMapPath${workingDir}/communityMap --isLookUpUrl${isLookUpUrl} @@ -176,6 +188,7 @@ --sourcePath${sourcePath}/otherresearchproduct --resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct --outputPath${workingDir}/otherresearchproduct + --communityMapPath${workingDir}/communityMap --isLookUpUrl${isLookUpUrl} @@ -202,6 +215,7 @@ --sourcePath${sourcePath}/software --resultTableNameeu.dnetlib.dhp.schema.oaf.Software --outputPath${workingDir}/software + --communityMapPath${workingDir}/communityMap --isLookUpUrl${isLookUpUrl} @@ -346,7 +360,7 @@ yarn cluster Split dumped result for community - eu.dnetlib.dhp.oa.graph.dump.community.SparkSplitForCommunity + eu.dnetlib.dhp.oa.graph.dump.community.SparkSplitForCommunityS3 dhp-graph-mapper-${projectVersion}.jar --executor-memory=${sparkExecutorMemory} @@ -360,9 +374,10 @@ --sourcePath${workingDir}/ext --outputPath${outputPath} + --communityMapPath${workingDir}/communityMap --isLookUpUrl${isLookUpUrl} - + @@ -374,6 +389,7 @@ --accessToken${accessToken} --connectionUrl${connectionUrl} --metadata${metadata} + --communityMapPath${workingDir}/communityMap --isLookUpUrl${isLookUpUrl} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json index 60fa7a95f1..36e72eb843 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json @@ -1,5 +1,17 @@ [ + { + "paramName":"is", + "paramLongName":"isLookUpUrl", + "paramDescription": "URL of the isLookUp Service", + "paramRequired": true + }, + { + "paramName":"cmp", + "paramLongName":"communityMapPath", + "paramDescription": "the path to the serialization of the community map", + "paramRequired": false + }, { "paramName":"is", "paramLongName":"isLookUpUrl", diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json index d65df95848..0e527852b1 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json @@ -1,11 +1,17 @@ [ -{ -"paramName":"is", -"paramLongName":"isLookUpUrl", -"paramDescription": "URL of the isLookUp Service", -"paramRequired": true -}, + { + "paramName":"is", + "paramLongName":"isLookUpUrl", + "paramDescription": "URL of the isLookUp Service", + "paramRequired": true + }, + { + "paramName":"cmp", + "paramLongName":"communityMapPath", + "paramDescription": "the path to the serialization of the community map", + "paramRequired": false + }, { "paramName":"hdfsp", "paramLongName":"hdfsPath",