diff --git a/dhp-workflows/dhp-continuous-validation/pom.xml b/dhp-workflows/dhp-continuous-validation/pom.xml index 1a834e146..5eaecf2d1 100644 --- a/dhp-workflows/dhp-continuous-validation/pom.xml +++ b/dhp-workflows/dhp-continuous-validation/pom.xml @@ -15,13 +15,13 @@ - eu.dnetlib.dhp dhp-common ${project.version} + eu.dnetlib uoa-validator-engine2 @@ -32,33 +32,18 @@ spark-core_${scala.binary.version} - org.apache.spark spark-sql_${scala.binary.version} - - - com.google.code.gson gson - - \ No newline at end of file diff --git a/dhp-workflows/dhp-continuous-validation/src/main/java/eu/dnetlib/dhp/continuous_validator/ContinuousValidator.java b/dhp-workflows/dhp-continuous-validation/src/main/java/eu/dnetlib/dhp/continuous_validator/ContinuousValidator.java index 72e10bdb8..9cf676e76 100644 --- a/dhp-workflows/dhp-continuous-validation/src/main/java/eu/dnetlib/dhp/continuous_validator/ContinuousValidator.java +++ b/dhp-workflows/dhp-continuous-validation/src/main/java/eu/dnetlib/dhp/continuous_validator/ContinuousValidator.java @@ -4,14 +4,12 @@ package eu.dnetlib.dhp.continuous_validator; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.nio.charset.StandardCharsets; -import java.util.List; import java.util.Objects; import java.util.Optional; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Row; import org.apache.spark.sql.SaveMode; @@ -21,7 +19,6 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.validator2.validation.XMLApplicationProfile; import eu.dnetlib.validator2.validation.guideline.openaire.*; import eu.dnetlib.validator2.validation.utils.TestUtils; -import scala.Option; public class ContinuousValidator { @@ -30,8 +27,6 @@ public class ContinuousValidator { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ContinuousValidator.class); private static final String parametersFile = "input_continuous_validator_parameters.json"; - private static final boolean SHOULD_GET_ARGUMENTS_FROM_FILE = true; // It throws an error for now.. - public static void main(String[] args) { ArgumentApplicationParser parser = null; @@ -41,65 +36,48 @@ public class ContinuousValidator { String guidelines = null; String outputPath = null; - if (SHOULD_GET_ARGUMENTS_FROM_FILE) { - try { - String jsonConfiguration = IOUtils - .toString( - Objects - .requireNonNull( - ContinuousValidator.class - .getResourceAsStream("/eu/dnetlib/dhp/continuous_validator/" + parametersFile)), - StandardCharsets.UTF_8); + try { + String jsonConfiguration = IOUtils + .toString( + Objects + .requireNonNull( + ContinuousValidator.class + .getResourceAsStream("/eu/dnetlib/dhp/continuous_validator/" + parametersFile)), + StandardCharsets.UTF_8); - parser = new ArgumentApplicationParser(jsonConfiguration); - parser.parseArgument(args); - } catch (Exception e) { - logger.error("Error when parsing the parameters!", e); - return; - } + parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + } catch (Exception e) { + logger.error("Error when parsing the parameters!", e); + return; + } - isSparkSessionManaged = Optional - .ofNullable(parser.get("isSparkSessionManaged")) // This param is not mandatory, so it may be null. - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); + isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) // This param is not mandatory, so it may be null. + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); - logger.info("isSparkSessionManaged: {}", isSparkSessionManaged); - // This is needed to implement a unit test in which the spark session is created in the context of the - // unit test itself rather than inside the spark application" + logger.info("isSparkSessionManaged: {}", isSparkSessionManaged); + // This is needed to implement a unit test in which the spark session is created in the context of the + // unit test itself rather than inside the spark application" - parquet_file_path = parser.get("parquet_file_path"); - if (parquet_file_path == null) { - logger.error("The \"parquet_file_path\" was not retrieved from the parameters file: " + parametersFile); - return; - } + parquet_file_path = parser.get("parquet_file_path"); + if (parquet_file_path == null) { + logger.error("The \"parquet_file_path\" was not retrieved from the parameters file: " + parametersFile); + return; + } - guidelines = parser.get("openaire_guidelines"); - if (guidelines == null) { - logger - .error("The \"openaire_guidelines\" was not retrieved from the parameters file: " + parametersFile); - return; - } + guidelines = parser.get("openaire_guidelines"); + if (guidelines == null) { + logger + .error("The \"openaire_guidelines\" was not retrieved from the parameters file: " + parametersFile); + return; + } - outputPath = parser.get("outputPath"); - if (outputPath == null) { - logger.error("The \"outputPath\" was not retrieved from the parameters file: " + parametersFile); - return; - } - - sparkMaster = "local[*]"; - } else { - if (args.length != 4) { - String errorMsg = "Wrong number of arguments given! Please run the app like so: java -jar target/dhp-continuous-validation-1.0.0-SNAPSHOT.jar "; - System.err.println(errorMsg); - logger.error(errorMsg); - System.exit(1); - } - sparkMaster = args[0]; - logger.info("Will use this Spark master: \"" + sparkMaster + "\"."); - - parquet_file_path = args[1]; - guidelines = args[2]; - outputPath = args[3]; + outputPath = parser.get("outputPath"); + if (outputPath == null) { + logger.error("The \"outputPath\" was not retrieved from the parameters file: " + parametersFile); + return; } if (!outputPath.endsWith("/")) @@ -156,7 +134,6 @@ public class ContinuousValidator { .option("compression", "gzip") .mode(SaveMode.Overwrite) .json(finalOutputPath + RESULTS_FILE_NAME); // The filename should be the name of the input-file or the - }); } }