- Add the "installProject.sh" script.
- Show the Job-ID or potential deployment-error-logs, right after the deployment of the workflow. - Code polishing.
This commit is contained in:
parent
00644ef487
commit
ff47a941f5
|
@ -0,0 +1,10 @@
|
||||||
|
cd ../../
|
||||||
|
|
||||||
|
DEFAULT_PROFILE='' # It's the empty profile.
|
||||||
|
NEWER_VERSIONS_PROFILE='-Pscala-2.12'
|
||||||
|
CHOSEN_MAVEN_PROFILE=${DEFAULT_PROFILE}
|
||||||
|
|
||||||
|
mvn clean install ${CHOSEN_MAVEN_PROFILE} -Dmaven.test.skip=true
|
||||||
|
|
||||||
|
# We skip tests for all modules, since the take a big amount of time and some of them fail.
|
||||||
|
# Any test added to this module, will be executed in the "runOozieWorkflow.sh" script.
|
|
@ -7,3 +7,6 @@ CHOSEN_MAVEN_PROFILE=${DEFAULT_PROFILE}
|
||||||
|
|
||||||
mvn clean package ${CHOSEN_MAVEN_PROFILE} -Poozie-package,deploy,run \
|
mvn clean package ${CHOSEN_MAVEN_PROFILE} -Poozie-package,deploy,run \
|
||||||
-Dworkflow.source.dir=eu/dnetlib/dhp/continuous_validator
|
-Dworkflow.source.dir=eu/dnetlib/dhp/continuous_validator
|
||||||
|
|
||||||
|
echo -e "\n\nShowing the contents of \"extract-and-run-on-remote-host.log\":\n\n"
|
||||||
|
cat ./target/extract-and-run-on-remote-host.log
|
||||||
|
|
|
@ -1,21 +1,6 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.continuous_validator;
|
package eu.dnetlib.dhp.continuous_validator;
|
||||||
|
|
||||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Optional;
|
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
|
||||||
import org.apache.spark.SparkConf;
|
|
||||||
import org.apache.spark.api.java.function.MapFunction;
|
|
||||||
import org.apache.spark.sql.Encoders;
|
|
||||||
import org.apache.spark.sql.Row;
|
|
||||||
import org.apache.spark.sql.SaveMode;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
import eu.dnetlib.validator2.validation.StandardValidationResult;
|
import eu.dnetlib.validator2.validation.StandardValidationResult;
|
||||||
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
||||||
|
@ -23,18 +8,29 @@ import eu.dnetlib.validator2.validation.guideline.Guideline;
|
||||||
import eu.dnetlib.validator2.validation.guideline.StandardResult;
|
import eu.dnetlib.validator2.validation.guideline.StandardResult;
|
||||||
import eu.dnetlib.validator2.validation.guideline.openaire.*;
|
import eu.dnetlib.validator2.validation.guideline.openaire.*;
|
||||||
import eu.dnetlib.validator2.validation.utils.TestUtils;
|
import eu.dnetlib.validator2.validation.utils.TestUtils;
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.spark.SparkConf;
|
||||||
|
import org.apache.spark.api.java.function.MapFunction;
|
||||||
|
import org.apache.spark.sql.Encoders;
|
||||||
|
import org.apache.spark.sql.Row;
|
||||||
|
import org.apache.spark.sql.SaveMode;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||||
|
|
||||||
public class ContinuousValidator {
|
public class ContinuousValidator {
|
||||||
|
|
||||||
public static final String TEST_FILES_V4_DIR = TestUtils.TEST_FILES_BASE_DIR + "openaireguidelinesV4/";
|
|
||||||
public static final String RESULTS_FILE_NAME = "results.json";
|
|
||||||
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ContinuousValidator.class);
|
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ContinuousValidator.class);
|
||||||
private static final String parametersFile = "input_continuous_validator_parameters.json";
|
private static final String parametersFile = "input_continuous_validator_parameters.json";
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
|
|
||||||
ArgumentApplicationParser parser = null;
|
ArgumentApplicationParser parser = null;
|
||||||
String sparkMaster = null;
|
|
||||||
Boolean isSparkSessionManaged = false;
|
Boolean isSparkSessionManaged = false;
|
||||||
String parquet_file_path = null;
|
String parquet_file_path = null;
|
||||||
String guidelines = null;
|
String guidelines = null;
|
||||||
|
@ -91,7 +87,7 @@ public class ContinuousValidator {
|
||||||
logger
|
logger
|
||||||
.info(
|
.info(
|
||||||
"Will validate the contents of parquetFile: \"" + parquet_file_path + "\", against guidelines: \""
|
"Will validate the contents of parquetFile: \"" + parquet_file_path + "\", against guidelines: \""
|
||||||
+ guidelines + "\"" + " and will output the results in: " + outputPath + RESULTS_FILE_NAME);
|
+ guidelines + "\"" + " and will output the results in: " + outputPath);
|
||||||
|
|
||||||
AbstractOpenAireProfile profile;
|
AbstractOpenAireProfile profile;
|
||||||
switch (guidelines) {
|
switch (guidelines) {
|
||||||
|
@ -142,7 +138,7 @@ public class ContinuousValidator {
|
||||||
.write()
|
.write()
|
||||||
.option("compression", "gzip")
|
.option("compression", "gzip")
|
||||||
.mode(SaveMode.Overwrite)
|
.mode(SaveMode.Overwrite)
|
||||||
.json(finalOutputPath + RESULTS_FILE_NAME); // The filename should be the name of the input-file or the
|
.json(finalOutputPath); // The output files have names of the input files which were processed.
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue