From 8765020b85acf0f88a252197de04de4dbb9735fc Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Mon, 4 Jan 2021 17:37:08 +0100 Subject: [PATCH] minor fix --- .../doiboost/SparkGenerateDoiBoost.scala | 2 +- ...ProcessMAG.scala => SparkProcessMAG.scala} | 2 +- .../doiboost/generate_doiboost_params.json | 2 +- .../dhp/doiboost/oozie_app/workflow.xml | 20 ++++++++----------- 4 files changed, 11 insertions(+), 15 deletions(-) rename dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/{SparkPreProcessMAG.scala => SparkProcessMAG.scala} (99%) diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala index 19a512ac66..11f9828db4 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala @@ -31,7 +31,7 @@ object SparkGenerateDoiBoost { import spark.implicits._ val hostedByMapPath = parser.get("hostedByMapPath") - val workingDirPath = parser.get("workingDirPath") + val workingDirPath = parser.get("workingPath") implicit val mapEncoderPub: Encoder[Publication] = Encoders.kryo[Publication] diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkPreProcessMAG.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkProcessMAG.scala similarity index 99% rename from dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkPreProcessMAG.scala rename to dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkProcessMAG.scala index 02dc4979af..780e65c1e9 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkPreProcessMAG.scala +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkProcessMAG.scala @@ -11,7 +11,7 @@ import org.slf4j.{Logger, LoggerFactory} import scala.collection.JavaConverters._ -object SparkPreProcessMAG { +object SparkProcessMAG { def main(args: Array[String]): Unit = { val logger: Logger = LoggerFactory.getLogger(getClass) diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_params.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_params.json index ea08f47d43..1ff63dd0ec 100644 --- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_params.json +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_params.json @@ -3,5 +3,5 @@ {"paramName": "hb", "paramLongName":"hostedByMapPath", "paramDescription": "the hosted By Map Path", "paramRequired": true}, {"paramName": "ap", "paramLongName":"affiliationPath", "paramDescription": "the Affliation Path", "paramRequired": true}, {"paramName": "pa", "paramLongName":"paperAffiliationPath", "paramDescription": "the paperAffiliation Path", "paramRequired": true}, - {"paramName": "w", "paramLongName":"workingDirPath", "paramDescription": "the Working Path", "paramRequired": true} + {"paramName": "w", "paramLongName":"workingPath", "paramDescription": "the Working Path", "paramRequired": true} ] diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/oozie_app/workflow.xml index d779725129..c116d07b1d 100644 --- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/oozie_app/workflow.xml @@ -166,9 +166,9 @@ - - - + + + @@ -191,18 +191,16 @@ --targetPath${inputPathMAG}/dataset --masteryarn-cluster - + - - - + yarn-cluster cluster Convert Mag to OAF Dataset - eu.dnetlib.doiboost.mag.SparkPreProcessMAG + eu.dnetlib.doiboost.mag.SparkProcessMAG dhp-doiboost-${projectVersion}.jar --executor-memory=${sparkExecutorMemory} @@ -244,8 +242,6 @@ - - @@ -269,7 +265,7 @@ - + yarn-cluster @@ -287,7 +283,7 @@ --hostedByMapPath${hostedByMapPath} --affiliationPath${inputPathMAG}/process/Affiliations --paperAffiliationPath${inputPathMAG}/process/PaperAuthorAffiliations - --workingDirPath${workingPath} + --workingPath${workingPath} --masteryarn-cluster