diff --git a/dhp-build/dhp-build-assembly-resources/README.markdown b/dhp-build/dhp-build-assembly-resources/README.markdown new file mode 100644 index 000000000..a345dff12 --- /dev/null +++ b/dhp-build/dhp-build-assembly-resources/README.markdown @@ -0,0 +1,7 @@ +Module utilized by `dhp-workflows`. + +Contains all required resources by this parent module: + +* assembly XML definitions +* build shell scripts +* oozie package commands for uploading, running and monitoring oozie workflows diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml new file mode 100644 index 000000000..4273bc9db --- /dev/null +++ b/dhp-build/dhp-build-assembly-resources/pom.xml @@ -0,0 +1,25 @@ + + + + 4.0.0 + + + eu.dnetlib.dhp + dhp-build + 1.0.0-SNAPSHOT + + + dhp-build-assembly-resources + jar + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + + diff --git a/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/oozie-installer.xml b/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/oozie-installer.xml new file mode 100644 index 000000000..1419c5b1c --- /dev/null +++ b/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/oozie-installer.xml @@ -0,0 +1,32 @@ + + + oozie-installer + + dir + + + + + true + ${project.build.directory}/assembly-resources/commands + + / + + **/* + + 0755 + unix + + + / + diff --git a/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/tests.xml b/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/tests.xml new file mode 100644 index 000000000..bf679e652 --- /dev/null +++ b/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/tests.xml @@ -0,0 +1,24 @@ + + + tests + + jar + + false + + + ${project.build.testOutputDirectory} + + + + + + \ No newline at end of file diff --git a/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/get_working_dir.sh b/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/get_working_dir.sh new file mode 100644 index 000000000..e9d55f0d7 --- /dev/null +++ b/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/get_working_dir.sh @@ -0,0 +1,3 @@ +#!/bin/bash +hadoop fs -get ${workingDir} + diff --git a/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/print_working_dir.sh b/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/print_working_dir.sh new file mode 100644 index 000000000..c79839ea4 --- /dev/null +++ b/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/print_working_dir.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "" +echo "---->Contents of the working directory" +hadoop fs -ls ${workingDir} + diff --git a/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/readme.markdown b/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/readme.markdown new file mode 100644 index 000000000..3e049c18b --- /dev/null +++ b/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/readme.markdown @@ -0,0 +1,5 @@ +Execute the scripts in the following order: + +1. `upload_workflow.sh` +2. `run_workflow.sh` +3. `print_working_dir.sh` or `get_working_dir.sh` diff --git a/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/run_workflow.sh b/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/run_workflow.sh new file mode 100644 index 000000000..fee3d7737 --- /dev/null +++ b/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/run_workflow.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ $# = 0 ] ; then + oozie job -oozie ${oozieServiceLoc} -config job.properties -run +else + oozie job -oozie ${oozieServiceLoc} -config $1/job.properties -run +fi + + + diff --git a/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/upload_workflow.sh b/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/upload_workflow.sh new file mode 100644 index 000000000..c5d299c2f --- /dev/null +++ b/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/upload_workflow.sh @@ -0,0 +1,34 @@ +#!/bin/bash +exec 3>&1 +BASH_XTRACEFD=3 +set -x ## print every executed command + + +if [ $# = 0 ] ; then + target_dir_root=`pwd`'/${oozieAppDir}' +else + target_dir_root=`readlink -f $1`'/${oozieAppDir}' +fi + +# initial phase, creating symbolic links to jars in all subworkflows +# currently disabled +#libDir=$target_dir_root'/lib' +#dirs=`find $target_dir_root/* -maxdepth 10 -type d` +#for dir in $dirs +#do +# if [ -f $dir/workflow.xml ] +# then +# echo "creating symbolic links to jars in directory: $dir/lib" +# if [ ! -d "$dir/lib" ]; then +# mkdir $dir/lib +# fi +# find $libDir -type f -exec ln -s \{\} $dir/lib \; +# fi +#done + + +#uploading +hadoop fs -rm -r ${sandboxDir} +hadoop fs -mkdir -p ${sandboxDir} +hadoop fs -mkdir -p ${workingDir} +hadoop fs -put $target_dir_root ${sandboxDir} diff --git a/dhp-build/dhp-build-assembly-resources/src/main/resources/project-default.properties b/dhp-build/dhp-build-assembly-resources/src/main/resources/project-default.properties new file mode 100644 index 000000000..021ecf55b --- /dev/null +++ b/dhp-build/dhp-build-assembly-resources/src/main/resources/project-default.properties @@ -0,0 +1,7 @@ +#sandboxName when not provided explicitly will be generated +sandboxName=${sandboxName} +sandboxDir=/user/${iis.hadoop.frontend.user.name}/${sandboxName} +workingDir=${sandboxDir}/working_dir +oozie.wf.application.path = ${nameNode}${sandboxDir}/${oozieAppDir} +oozieTopWfApplicationPath = ${oozie.wf.application.path} + diff --git a/dhp-build/dhp-build-properties-maven-plugin/README.markdown b/dhp-build/dhp-build-properties-maven-plugin/README.markdown new file mode 100644 index 000000000..66234e871 --- /dev/null +++ b/dhp-build/dhp-build-properties-maven-plugin/README.markdown @@ -0,0 +1,6 @@ +Maven plugin module utilized by `dhp-workflows` for proper `job.properties` file building. + +It is based on http://site.kuali.org/maven/plugins/properties-maven-plugin/1.3.2/write-project-properties-mojo.html and supplemented with: + +* handling includePropertyKeysFromFiles property allowing writing only properties listed in given property files +As a final outcome only properties listed in `` element and listed as a keys in files from `` element will be written to output file. diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml new file mode 100644 index 000000000..eba085785 --- /dev/null +++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml @@ -0,0 +1,81 @@ + + + + 4.0.0 + + + eu.dnetlib.dhp + dhp-build + 1.0.0-SNAPSHOT + + + dhp-build-properties-maven-plugin + maven-plugin + + + + + org.apache.maven + maven-plugin-api + 2.0 + + + org.apache.maven + maven-project + 2.0 + + + org.kuali.maven.plugins + properties-maven-plugin + 1.3.2 + + + com.google.code.findbugs + annotations + 3.0.1 + provided + + + com.google.code.findbugs + jsr305 + 3.0.1 + provided + + + + + + target + target/classes + ${project.artifactId}-${project.version} + target/test-classes + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + verify + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + true + + + + + + diff --git a/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojo.java b/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojo.java new file mode 100644 index 000000000..389208e2b --- /dev/null +++ b/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojo.java @@ -0,0 +1,71 @@ +package eu.dnetlib.maven.plugin.properties; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; + +/** + * Generates oozie properties which were not provided from commandline. + * @author mhorst + * + * @goal generate-properties + */ +public class GenerateOoziePropertiesMojo extends AbstractMojo { + + public static final String PROPERTY_NAME_WF_SOURCE_DIR = "workflow.source.dir"; + public static final String PROPERTY_NAME_SANDBOX_NAME = "sandboxName"; + + private final String[] limiters = {"iis", "dnetlib", "eu"}; + + @Override + public void execute() throws MojoExecutionException, MojoFailureException { + if (System.getProperties().containsKey(PROPERTY_NAME_WF_SOURCE_DIR) && + !System.getProperties().containsKey(PROPERTY_NAME_SANDBOX_NAME)) { + String generatedSandboxName = generateSandboxName(System.getProperties().getProperty( + PROPERTY_NAME_WF_SOURCE_DIR)); + if (generatedSandboxName!=null) { + System.getProperties().setProperty(PROPERTY_NAME_SANDBOX_NAME, + generatedSandboxName); + } else { + System.out.println("unable to generate sandbox name from path: " + + System.getProperties().getProperty(PROPERTY_NAME_WF_SOURCE_DIR)); + } + } + } + + /** + * Generates sandbox name from workflow source directory. + * @param wfSourceDir + * @return generated sandbox name + */ + private String generateSandboxName(String wfSourceDir) { +// utilize all dir names until finding one of the limiters + List sandboxNameParts = new ArrayList(); + String[] tokens = StringUtils.split(wfSourceDir, File.separatorChar); + ArrayUtils.reverse(tokens); + if (tokens.length>0) { + for (String token : tokens) { + for (String limiter : limiters) { + if (limiter.equals(token)) { + return sandboxNameParts.size()>0? + StringUtils.join(sandboxNameParts.toArray()):null; + } + } + if (sandboxNameParts.size()>0) { + sandboxNameParts.add(0, File.separator); + } + sandboxNameParts.add(0, token); + } + return StringUtils.join(sandboxNameParts.toArray()); + } else { + return null; + } + } + +} diff --git a/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectProperties.java b/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectProperties.java new file mode 100644 index 000000000..62f04761a --- /dev/null +++ b/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectProperties.java @@ -0,0 +1,436 @@ +/** + * + * Licensed under the Educational Community License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.opensource.org/licenses/ecl2.php + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package eu.dnetlib.maven.plugin.properties; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.project.MavenProject; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * Writes project properties for the keys listed in specified properties files. + * Based on: + * http://site.kuali.org/maven/plugins/properties-maven-plugin/1.3.2/write-project-properties-mojo.html + + * @author mhorst + * @goal write-project-properties + */ +public class WritePredefinedProjectProperties extends AbstractMojo { + + private static final String CR = "\r"; + private static final String LF = "\n"; + private static final String TAB = "\t"; + protected static final String PROPERTY_PREFIX_ENV = "env."; + private static final String ENCODING_UTF8 = "utf8"; + + /** + * @parameter property="properties.includePropertyKeysFromFiles" + */ + private String[] includePropertyKeysFromFiles; + + /** + * @parameter default-value="${project}" + * @required + * @readonly + */ + protected MavenProject project; + + /** + * The file that properties will be written to + * + * @parameter property="properties.outputFile" + * default-value="${project.build.directory}/properties/project.properties"; + * @required + */ + protected File outputFile; + + /** + * If true, the plugin will silently ignore any non-existent properties files, and the build will continue + * + * @parameter property="properties.quiet" default-value="true" + */ + private boolean quiet; + + /** + * Comma separated list of characters to escape when writing property values. cr=carriage return, lf=linefeed, + * tab=tab. Any other values are taken literally. + * + * @parameter default-value="cr,lf,tab" property="properties.escapeChars" + */ + private String escapeChars; + + /** + * If true, the plugin will include system properties when writing the properties file. System properties override + * both environment variables and project properties. + * + * @parameter default-value="false" property="properties.includeSystemProperties" + */ + private boolean includeSystemProperties; + + /** + * If true, the plugin will include environment variables when writing the properties file. Environment variables + * are prefixed with "env". Environment variables override project properties. + * + * @parameter default-value="false" property="properties.includeEnvironmentVariables" + */ + private boolean includeEnvironmentVariables; + + /** + * Comma separated set of properties to exclude when writing the properties file + * + * @parameter property="properties.exclude" + */ + private String exclude; + + /** + * Comma separated set of properties to write to the properties file. If provided, only the properties matching + * those supplied here will be written to the properties file. + * + * @parameter property="properties.include" + */ + private String include; + + /* (non-Javadoc) + * @see org.apache.maven.plugin.AbstractMojo#execute() + */ + @Override + @SuppressFBWarnings({"NP_UNWRITTEN_FIELD","UWF_UNWRITTEN_FIELD"}) + public void execute() throws MojoExecutionException, MojoFailureException { + Properties properties = new Properties(); + // Add project properties + properties.putAll(project.getProperties()); + if (includeEnvironmentVariables) { + // Add environment variables, overriding any existing properties with the same key + properties.putAll(getEnvironmentVariables()); + } + if (includeSystemProperties) { + // Add system properties, overriding any existing properties with the same key + properties.putAll(System.getProperties()); + } + + // Remove properties as appropriate + trim(properties, exclude, include); + + String comment = "# " + new Date() + "\n"; + List escapeTokens = getEscapeChars(escapeChars); + + getLog().info("Creating " + outputFile); + writeProperties(outputFile, comment, properties, escapeTokens); + } + + /** + * Provides environment variables. + * @return environment variables + */ + protected static Properties getEnvironmentVariables() { + Properties props = new Properties(); + for (Entry entry : System.getenv().entrySet()) { + props.setProperty(PROPERTY_PREFIX_ENV + entry.getKey(), entry.getValue()); + } + return props; + } + + /** + * Removes properties which should not be written. + * @param properties + * @param omitCSV + * @param includeCSV + * @throws MojoExecutionException + */ + protected void trim(Properties properties, String omitCSV, String includeCSV) throws MojoExecutionException { + List omitKeys = getListFromCSV(omitCSV); + for (String key : omitKeys) { + properties.remove(key); + } + + List includeKeys = getListFromCSV(includeCSV); +// mh: including keys from predefined properties + if (includePropertyKeysFromFiles!=null && includePropertyKeysFromFiles.length>0) { + for (String currentIncludeLoc : includePropertyKeysFromFiles) { + if (validate(currentIncludeLoc)) { + Properties p = getProperties(currentIncludeLoc); + for (String key : p.stringPropertyNames()) { + includeKeys.add(key); + } + } + } + } + if (includeKeys!=null && !includeKeys.isEmpty()) { +// removing only when include keys provided + Set keys = properties.stringPropertyNames(); + for (String key : keys) { + if (!includeKeys.contains(key)) { + properties.remove(key); + } + } + } + } + + /** + * Checks whether file exists. + * @param location + * @return true when exists, false otherwise. + */ + protected boolean exists(String location) { + if (StringUtils.isBlank(location)) { + return false; + } + File file = new File(location); + if (file.exists()) { + return true; + } + ResourceLoader loader = new DefaultResourceLoader(); + Resource resource = loader.getResource(location); + return resource.exists(); + } + + /** + * Validates resource location. + * @param location + * @return true when valid, false otherwise + * @throws MojoExecutionException + */ + protected boolean validate(String location) throws MojoExecutionException { + boolean exists = exists(location); + if (exists) { + return true; + } + if (quiet) { + getLog().info("Ignoring non-existent properties file '" + location + "'"); + return false; + } else { + throw new MojoExecutionException("Non-existent properties file '" + location + "'"); + } + } + + /** + * Provides input stream. + * @param location + * @return input stream + * @throws IOException + */ + protected InputStream getInputStream(String location) throws IOException { + File file = new File(location); + if (file.exists()) { + return new FileInputStream(location); + } + ResourceLoader loader = new DefaultResourceLoader(); + Resource resource = loader.getResource(location); + return resource.getInputStream(); + } + + /** + * Creates properties for given location. + * @param location + * @return properties for given location + * @throws MojoExecutionException + */ + protected Properties getProperties(String location) throws MojoExecutionException { + InputStream in = null; + try { + Properties properties = new Properties(); + in = getInputStream(location); + if (location.toLowerCase().endsWith(".xml")) { + properties.loadFromXML(in); + } else { + properties.load(in); + } + return properties; + } catch (IOException e) { + throw new MojoExecutionException("Error reading properties file " + location, e); + } finally { + IOUtils.closeQuietly(in); + } + } + + /** + * Provides escape characters. + * @param escapeChars + * @return escape characters + */ + protected List getEscapeChars(String escapeChars) { + List tokens = getListFromCSV(escapeChars); + List realTokens = new ArrayList(); + for (String token : tokens) { + String realToken = getRealToken(token); + realTokens.add(realToken); + } + return realTokens; + } + + /** + * Provides real token. + * @param token + * @return real token + */ + protected String getRealToken(String token) { + if (token.equalsIgnoreCase("CR")) { + return CR; + } else if (token.equalsIgnoreCase("LF")) { + return LF; + } else if (token.equalsIgnoreCase("TAB")) { + return TAB; + } else { + return token; + } + } + + /** + * Returns content. + * @param comment + * @param properties + * @param escapeTokens + * @return content + */ + protected String getContent(String comment, Properties properties, List escapeTokens) { + List names = new ArrayList(properties.stringPropertyNames()); + Collections.sort(names); + StringBuilder sb = new StringBuilder(); + if (!StringUtils.isBlank(comment)) { + sb.append(comment); + } + for (String name : names) { + String value = properties.getProperty(name); + String escapedValue = escape(value, escapeTokens); + sb.append(name + "=" + escapedValue + "\n"); + } + return sb.toString(); + } + + /** + * Writes properties to given file. + * @param file + * @param comment + * @param properties + * @param escapeTokens + * @throws MojoExecutionException + */ + protected void writeProperties(File file, String comment, Properties properties, List escapeTokens) + throws MojoExecutionException { + try { + String content = getContent(comment, properties, escapeTokens); + FileUtils.writeStringToFile(file, content, ENCODING_UTF8); + } catch (IOException e) { + throw new MojoExecutionException("Error creating properties file", e); + } + } + + /** + * Escapes characters. + * @param s + * @param escapeChars + * @return + */ + protected String escape(String s, List escapeChars) { + String result = s; + for (String escapeChar : escapeChars) { + result = result.replace(escapeChar, getReplacementToken(escapeChar)); + } + return result; + } + + /** + * Provides replacement token. + * @param escapeChar + * @return replacement token + */ + protected String getReplacementToken(String escapeChar) { + if (escapeChar.equals(CR)) { + return "\\r"; + } else if (escapeChar.equals(LF)) { + return "\\n"; + } else if (escapeChar.equals(TAB)) { + return "\\t"; + } else { + return "\\" + escapeChar; + } + } + + /** + * Returns list from csv. + * @param csv + * @return list of values generated from CSV + */ + protected static final List getListFromCSV(String csv) { + if (StringUtils.isBlank(csv)) { + return new ArrayList(); + } + List list = new ArrayList(); + String[] tokens = StringUtils.split(csv, ","); + for (String token : tokens) { + list.add(token.trim()); + } + return list; + } + + public void setIncludeSystemProperties(boolean includeSystemProperties) { + this.includeSystemProperties = includeSystemProperties; + } + + public void setEscapeChars(String escapeChars) { + this.escapeChars = escapeChars; + } + + public void setIncludeEnvironmentVariables(boolean includeEnvironmentVariables) { + this.includeEnvironmentVariables = includeEnvironmentVariables; + } + + public void setExclude(String exclude) { + this.exclude = exclude; + } + + public void setInclude(String include) { + this.include = include; + } + + public void setQuiet(boolean quiet) { + this.quiet = quiet; + } + + /** + * Sets property files for which keys properties should be included. + * @param includePropertyKeysFromFiles + */ + public void setIncludePropertyKeysFromFiles( + String[] includePropertyKeysFromFiles) { + if (includePropertyKeysFromFiles!=null) { + this.includePropertyKeysFromFiles = Arrays.copyOf( + includePropertyKeysFromFiles, + includePropertyKeysFromFiles.length); + } + } + +} \ No newline at end of file diff --git a/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojoTest.java b/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojoTest.java new file mode 100644 index 000000000..8a763c1bd --- /dev/null +++ b/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojoTest.java @@ -0,0 +1,101 @@ +package eu.dnetlib.maven.plugin.properties; + +import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_SANDBOX_NAME; +import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_WF_SOURCE_DIR; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Before; +import org.junit.Test; + +/** + * @author mhorst + * + */ +public class GenerateOoziePropertiesMojoTest { + + private GenerateOoziePropertiesMojo mojo = new GenerateOoziePropertiesMojo(); + + @Before + public void clearSystemProperties() { + System.clearProperty(PROPERTY_NAME_SANDBOX_NAME); + System.clearProperty(PROPERTY_NAME_WF_SOURCE_DIR); + } + + @Test + public void testExecuteEmpty() throws Exception { + // execute + mojo.execute(); + + // assert + assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME)); + } + + @Test + public void testExecuteSandboxNameAlreadySet() throws Exception { + // given + String workflowSourceDir = "eu/dnetlib/iis/wf/transformers"; + String sandboxName = "originalSandboxName"; + System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir); + System.setProperty(PROPERTY_NAME_SANDBOX_NAME, sandboxName); + + // execute + mojo.execute(); + + // assert + assertEquals(sandboxName, System.getProperty(PROPERTY_NAME_SANDBOX_NAME)); + } + + @Test + public void testExecuteEmptyWorkflowSourceDir() throws Exception { + // given + String workflowSourceDir = ""; + System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir); + + // execute + mojo.execute(); + + // assert + assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME)); + } + + @Test + public void testExecuteNullSandboxNameGenerated() throws Exception { + // given + String workflowSourceDir = "eu/dnetlib/iis/"; + System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir); + + // execute + mojo.execute(); + + // assert + assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME)); + } + + @Test + public void testExecute() throws Exception { + // given + String workflowSourceDir = "eu/dnetlib/iis/wf/transformers"; + System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir); + + // execute + mojo.execute(); + + // assert + assertEquals("wf/transformers", System.getProperty(PROPERTY_NAME_SANDBOX_NAME)); + } + + @Test + public void testExecuteWithoutRoot() throws Exception { + // given + String workflowSourceDir = "wf/transformers"; + System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir); + + // execute + mojo.execute(); + + // assert + assertEquals("wf/transformers", System.getProperty(PROPERTY_NAME_SANDBOX_NAME)); + } + +} diff --git a/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java b/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java new file mode 100644 index 000000000..51d9575ff --- /dev/null +++ b/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java @@ -0,0 +1,365 @@ +package eu.dnetlib.maven.plugin.properties; + +import static eu.dnetlib.maven.plugin.properties.WritePredefinedProjectProperties.PROPERTY_PREFIX_ENV; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Properties; + +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.project.MavenProject; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + + +/** + * @author mhorst + * + */ +@RunWith(MockitoJUnitRunner.class) +public class WritePredefinedProjectPropertiesTest { + + @Rule + public TemporaryFolder testFolder = new TemporaryFolder(); + + @Mock + private MavenProject mavenProject; + + private WritePredefinedProjectProperties mojo; + + @Before + public void init() { + mojo = new WritePredefinedProjectProperties(); + mojo.outputFile = getPropertiesFileLocation(); + mojo.project = mavenProject; + doReturn(new Properties()).when(mavenProject).getProperties(); + } + + // ----------------------------------- TESTS --------------------------------------------- + + @Test + public void testExecuteEmpty() throws Exception { + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertEquals(0, storedProperties.size()); + } + + @Test + public void testExecuteWithProjectProperties() throws Exception { + // given + String key = "projectPropertyKey"; + String value = "projectPropertyValue"; + Properties projectProperties = new Properties(); + projectProperties.setProperty(key, value); + doReturn(projectProperties).when(mavenProject).getProperties(); + + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertEquals(1, storedProperties.size()); + assertTrue(storedProperties.containsKey(key)); + assertEquals(value, storedProperties.getProperty(key)); + } + + @Test(expected=MojoExecutionException.class) + public void testExecuteWithProjectPropertiesAndInvalidOutputFile() throws Exception { + // given + String key = "projectPropertyKey"; + String value = "projectPropertyValue"; + Properties projectProperties = new Properties(); + projectProperties.setProperty(key, value); + doReturn(projectProperties).when(mavenProject).getProperties(); + mojo.outputFile = testFolder.getRoot(); + + // execute + mojo.execute(); + } + + @Test + public void testExecuteWithProjectPropertiesExclusion() throws Exception { + // given + String key = "projectPropertyKey"; + String value = "projectPropertyValue"; + String excludedKey = "excludedPropertyKey"; + String excludedValue = "excludedPropertyValue"; + Properties projectProperties = new Properties(); + projectProperties.setProperty(key, value); + projectProperties.setProperty(excludedKey, excludedValue); + doReturn(projectProperties).when(mavenProject).getProperties(); + mojo.setExclude(excludedKey); + + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertEquals(1, storedProperties.size()); + assertTrue(storedProperties.containsKey(key)); + assertEquals(value, storedProperties.getProperty(key)); + } + + @Test + public void testExecuteWithProjectPropertiesInclusion() throws Exception { + // given + String key = "projectPropertyKey"; + String value = "projectPropertyValue"; + String includedKey = "includedPropertyKey"; + String includedValue = "includedPropertyValue"; + Properties projectProperties = new Properties(); + projectProperties.setProperty(key, value); + projectProperties.setProperty(includedKey, includedValue); + doReturn(projectProperties).when(mavenProject).getProperties(); + mojo.setInclude(includedKey); + + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertEquals(1, storedProperties.size()); + assertTrue(storedProperties.containsKey(includedKey)); + assertEquals(includedValue, storedProperties.getProperty(includedKey)); + } + + @Test + public void testExecuteIncludingPropertyKeysFromFile() throws Exception { + // given + String key = "projectPropertyKey"; + String value = "projectPropertyValue"; + String includedKey = "includedPropertyKey"; + String includedValue = "includedPropertyValue"; + Properties projectProperties = new Properties(); + projectProperties.setProperty(key, value); + projectProperties.setProperty(includedKey, includedValue); + doReturn(projectProperties).when(mavenProject).getProperties(); + + File includedPropertiesFile = new File(testFolder.getRoot(), "included.properties"); + Properties includedProperties = new Properties(); + includedProperties.setProperty(includedKey, "irrelevantValue"); + includedProperties.store(new FileWriter(includedPropertiesFile), null); + + mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()}); + + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertEquals(1, storedProperties.size()); + assertTrue(storedProperties.containsKey(includedKey)); + assertEquals(includedValue, storedProperties.getProperty(includedKey)); + } + + @Test + public void testExecuteIncludingPropertyKeysFromClasspathResource() throws Exception { + // given + String key = "projectPropertyKey"; + String value = "projectPropertyValue"; + String includedKey = "includedPropertyKey"; + String includedValue = "includedPropertyValue"; + Properties projectProperties = new Properties(); + projectProperties.setProperty(key, value); + projectProperties.setProperty(includedKey, includedValue); + doReturn(projectProperties).when(mavenProject).getProperties(); + + mojo.setIncludePropertyKeysFromFiles(new String[] {"/eu/dnetlib/maven/plugin/properties/included.properties"}); + + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertEquals(1, storedProperties.size()); + assertTrue(storedProperties.containsKey(includedKey)); + assertEquals(includedValue, storedProperties.getProperty(includedKey)); + } + + @Test(expected=MojoExecutionException.class) + public void testExecuteIncludingPropertyKeysFromBlankLocation() throws Exception { + // given + String key = "projectPropertyKey"; + String value = "projectPropertyValue"; + String includedKey = "includedPropertyKey"; + String includedValue = "includedPropertyValue"; + Properties projectProperties = new Properties(); + projectProperties.setProperty(key, value); + projectProperties.setProperty(includedKey, includedValue); + doReturn(projectProperties).when(mavenProject).getProperties(); + + mojo.setIncludePropertyKeysFromFiles(new String[] {""}); + + // execute + mojo.execute(); + } + + @Test + public void testExecuteIncludingPropertyKeysFromXmlFile() throws Exception { + // given + String key = "projectPropertyKey"; + String value = "projectPropertyValue"; + String includedKey = "includedPropertyKey"; + String includedValue = "includedPropertyValue"; + Properties projectProperties = new Properties(); + projectProperties.setProperty(key, value); + projectProperties.setProperty(includedKey, includedValue); + doReturn(projectProperties).when(mavenProject).getProperties(); + + File includedPropertiesFile = new File(testFolder.getRoot(), "included.xml"); + Properties includedProperties = new Properties(); + includedProperties.setProperty(includedKey, "irrelevantValue"); + includedProperties.storeToXML(new FileOutputStream(includedPropertiesFile), null); + + mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()}); + + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertEquals(1, storedProperties.size()); + assertTrue(storedProperties.containsKey(includedKey)); + assertEquals(includedValue, storedProperties.getProperty(includedKey)); + } + + @Test(expected=MojoExecutionException.class) + public void testExecuteIncludingPropertyKeysFromInvalidXmlFile() throws Exception { + // given + String key = "projectPropertyKey"; + String value = "projectPropertyValue"; + String includedKey = "includedPropertyKey"; + String includedValue = "includedPropertyValue"; + Properties projectProperties = new Properties(); + projectProperties.setProperty(key, value); + projectProperties.setProperty(includedKey, includedValue); + doReturn(projectProperties).when(mavenProject).getProperties(); + + File includedPropertiesFile = new File(testFolder.getRoot(), "included.xml"); + Properties includedProperties = new Properties(); + includedProperties.setProperty(includedKey, "irrelevantValue"); + includedProperties.store(new FileOutputStream(includedPropertiesFile), null); + + mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()}); + + // execute + mojo.execute(); + } + + @Test + public void testExecuteWithQuietModeOn() throws Exception { + // given + mojo.setQuiet(true); + mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"}); + + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertEquals(0, storedProperties.size()); + } + + @Test(expected=MojoExecutionException.class) + public void testExecuteIncludingPropertyKeysFromInvalidFile() throws Exception { + // given + mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"}); + + // execute + mojo.execute(); + } + + @Test + public void testExecuteWithEnvironmentProperties() throws Exception { + // given + mojo.setIncludeEnvironmentVariables(true); + + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertTrue(storedProperties.size() > 0); + for (Object currentKey : storedProperties.keySet()) { + assertTrue(((String)currentKey).startsWith(PROPERTY_PREFIX_ENV)); + } + } + + @Test + public void testExecuteWithSystemProperties() throws Exception { + // given + String key = "systemPropertyKey"; + String value = "systemPropertyValue"; + System.setProperty(key, value); + mojo.setIncludeSystemProperties(true); + + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertTrue(storedProperties.size() > 0); + assertTrue(storedProperties.containsKey(key)); + assertEquals(value, storedProperties.getProperty(key)); + } + + @Test + public void testExecuteWithSystemPropertiesAndEscapeChars() throws Exception { + // given + String key = "systemPropertyKey "; + String value = "systemPropertyValue"; + System.setProperty(key, value); + mojo.setIncludeSystemProperties(true); + String escapeChars = "cr,lf,tab,|"; + mojo.setEscapeChars(escapeChars); + + // execute + mojo.execute(); + + // assert + assertTrue(mojo.outputFile.exists()); + Properties storedProperties = getStoredProperties(); + assertTrue(storedProperties.size() > 0); + assertFalse(storedProperties.containsKey(key)); + assertTrue(storedProperties.containsKey(key.trim())); + assertEquals(value, storedProperties.getProperty(key.trim())); + } + + // ----------------------------------- PRIVATE ------------------------------------------- + + private File getPropertiesFileLocation() { + return new File(testFolder.getRoot(), "test.properties"); + } + + private Properties getStoredProperties() throws FileNotFoundException, IOException { + Properties properties = new Properties(); + properties.load(new FileInputStream(getPropertiesFileLocation())); + return properties; + } +} diff --git a/dhp-build/dhp-build-properties-maven-plugin/src/test/resources/eu/dnetlib/maven/plugin/properties/included.properties b/dhp-build/dhp-build-properties-maven-plugin/src/test/resources/eu/dnetlib/maven/plugin/properties/included.properties new file mode 100644 index 000000000..3c79fe6cb --- /dev/null +++ b/dhp-build/dhp-build-properties-maven-plugin/src/test/resources/eu/dnetlib/maven/plugin/properties/included.properties @@ -0,0 +1 @@ +includedPropertyKey=irrelevantValue \ No newline at end of file diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml index bc95d979a..a930af4ea 100644 --- a/dhp-build/pom.xml +++ b/dhp-build/pom.xml @@ -9,7 +9,8 @@ dhp-build pom - + dhp-build-assembly-resources + dhp-build-properties-maven-plugin diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/oozie/workflows/oozie_collection_workflows.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/oozie_app/oozie_collection_workflows.xml similarity index 100% rename from dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/oozie/workflows/oozie_collection_workflows.xml rename to dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/oozie_app/oozie_collection_workflows.xml diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/oozie/workflows/oozie_transform_workflows.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/transformation/oozie_app/oozie_transform_workflows.xml similarity index 100% rename from dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/oozie/workflows/oozie_transform_workflows.xml rename to dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/transformation/oozie_app/oozie_transform_workflows.xml diff --git a/dhp-workflows/dhp-distcp/pom.xml b/dhp-workflows/dhp-distcp/pom.xml new file mode 100644 index 000000000..5964c2976 --- /dev/null +++ b/dhp-workflows/dhp-distcp/pom.xml @@ -0,0 +1,15 @@ + + + + dhp-workflows + eu.dnetlib.dhp + 1.0.0-SNAPSHOT + + 4.0.0 + + dhp-distcp + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/config-default.xml b/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/config-default.xml new file mode 100644 index 000000000..905fb9984 --- /dev/null +++ b/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/config-default.xml @@ -0,0 +1,18 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + sourceNN + webhdfs://namenode2.hadoop.dm.openaire.eu:50071 + + + oozie.use.system.libpath + true + + \ No newline at end of file diff --git a/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/workflow.xml b/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/workflow.xml new file mode 100644 index 000000000..91b97332b --- /dev/null +++ b/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/workflow.xml @@ -0,0 +1,46 @@ + + + + sourceNN + the source name node + + + sourcePath + the source path + + + targetPath + the target path + + + hbase_dump_distcp_memory_mb + 6144 + memory for distcp action copying InfoSpace dump from remote cluster + + + hbase_dump_distcp_num_maps + 1 + maximum number of simultaneous copies of InfoSpace dump from remote location + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + -Dmapreduce.map.memory.mb=${hbase_dump_distcp_memory_mb} + -pb + -m ${hbase_dump_distcp_num_maps} + ${sourceNN}/${sourcePath} + ${nameNode}/${targetPath} + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index f8fcde2a8..fe0aabd3b 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -1,16 +1,604 @@ - + + 4.0.0 + eu.dnetlib.dhp dhp 1.0.0-SNAPSHOT ../ + dhp-workflows pom + dhp-aggregation + dhp-distcp + + + iis-releases + iis releases plugin repository + http://maven.ceon.pl/artifactory/iis-releases + default + + + + yyyy-MM-dd_HH_mm + + + oozie-package + + src/test/resources/define/path/pointing/to/directory/holding/oozie_app + oozie_app + default + default + default + primed + + runtime + + true + + ${user.home}/.dhp/application.properties + + ${maven.build.timestamp} + + ${project.version} + true + + + + eu.dnetlib.dhp + dhp-build-assembly-resources + ${project.version} + + + org.apache.oozie + oozie-client + + + net.schmizz + sshj + test + + + + + + attach-test-resources + + + + provided + + false + + + + oozie-package + + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.4.1 + + + enforce-connection-properties-file-existence + initialize + + enforce + + + + + + ${dhpConnectionProperties} + + + The file with connection properties could not be found. Please, create the ${dhpConnectionProperties} file or set the location to another already created file by using + -DdhpConnectionProperties property. + + + + true + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy dependencies + prepare-package + + copy-dependencies + + + ${oozie.package.dependencies.include.scope} + ${oozie.package.dependencies.exclude.scope} + true + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + attach-test-resources-package + prepare-package + + test-jar + + + ${oozie.package.skip.test.jar} + + + + + + + eu.dnetlib.primer + primer-maven-plugin + + + priming + prepare-package + + prime + + + + ${project.build.directory}/dependency/*.jar + ${project.build.directory}/*-tests.jar + ${project.build.directory}/classes + + ${project.build.directory}/dependency + ${project.build.directory}/${primed.dir} + ${workflow.source.dir} + + + + + + + org.kuali.maven.plugins + properties-maven-plugin + 1.3.2 + + + eu.dnetlib.dhp + dhp-build-assembly-resources + ${project.version} + + + + + + reading-dhp-properties + initialize + + read-project-properties + + + + ${dhpConnectionProperties} + + false + + + + read-default-properties + prepare-package + + read-project-properties + + + + classpath:project-default.properties + + true + + + + read-job-properties + prepare-package + + read-project-properties + + + + ${project.build.directory}/${primed.dir}/job.properties + job-override.properties + + true + + + + + + eu.dnetlib.dhp + dhp-build-properties-maven-plugin + + + validate + + generate-properties + + + + + + + write-job-properties + prepare-package + + write-project-properties + + + target/${oozie.package.file.name}/job.properties + + nameNode,jobTracker,queueName,importerQueueName,oozieLauncherQueueName, + workingDir,oozieTopWfApplicationPath,oozieServiceLoc, + sparkDriverMemory,sparkExecutorMemory,sparkExecutorCores, + oozie.wf.application.path,projectVersion,oozie.use.system.libpath + true + + + ${project.build.directory}/${primed.dir}/job.properties + job-override.properties + + + + + + + + pl.project13.maven + git-commit-id-plugin + 2.1.11 + + + + revision + + + + + true + yyyy-MM-dd'T'HH:mm:ssZ + true + target/${oozie.package.file.name}/${oozieAppDir}/version.properties + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.0.0 + + + eu.dnetlib.dhp + dhp-build-assembly-resources + ${project.version} + + + + + assembly-oozie-installer + package + + single + + + false + ${oozie.package.file.name}_shell_scripts + + oozie-installer + + + + + + + + + + maven-antrun-plugin + + + + installer-copy-custom + process-resources + + run + + + + + + + + + + + package + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + run + + + + + + + + + deploy + + + + org.codehaus.mojo + exec-maven-plugin + 1.5.0 + + + create-target-dir + package + + exec + + + ssh + + ${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name} + -p ${dhp.hadoop.frontend.port.ssh} + -o StrictHostKeyChecking=no + rm -rf ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; mkdir -p ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/ + + + + + upload-oozie-package + package + + exec + + + scp + + -P ${dhp.hadoop.frontend.port.ssh} + -o StrictHostKeyChecking=no + target/${oozie.package.file.name}.tar.gz + ${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}:${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/${oozie.package.file.name}.tar.gz + + + + + extract-and-upload-to-hdfs + package + + exec + + + ssh + + + ${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name} + -p ${dhp.hadoop.frontend.port.ssh} + -o StrictHostKeyChecking=no + cd ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; + tar -zxf oozie-package.tar.gz; + rm ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/oozie-package.tar.gz; + ./upload_workflow.sh + + + + + + + + + + run + + + + org.codehaus.mojo + exec-maven-plugin + 1.5.0 + + + run-job + package + + exec + + + ssh + + ${oozie.execution.log.file.location} + + ${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name} + -p ${dhp.hadoop.frontend.port.ssh} + -o StrictHostKeyChecking=no + cd ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; + ./run_workflow.sh + + + + + show-run-log-on-stdout + package + + exec + + + cat + + ${oozie.execution.log.file.location} + + + + + + + + + + + child-tests + + + + org.apache.maven.plugins + maven-jar-plugin + + + integration-test-package + integration-test + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + eu.dnetlib.iis.common.IntegrationTest + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + default-integration-test + + -Xmx1024m + + + ${dhpConnectionProperties} + ${output.dir.name} + + + + + + + + + + src/main/resources/parent.marker + + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.kuali.maven.plugins + + + properties-maven-plugin + + + [1.3.2,) + + + + read-project-properties + + + write-project-properties + + + + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + [1.0.0,) + + copy-dependencies + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 46386caf9..9d83791f0 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ - + dhp-build dhp-common dhp-workflows dhp-applications @@ -60,6 +60,8 @@ true + + cloudera @@ -142,6 +145,26 @@ commons-cli 1.4 + + + net.schmizz + sshj + 0.10.0 + + + + org.apache.oozie + oozie-client + ${dhp.oozie.version} + + + + slf4j-simple + org.slf4j + + + + @@ -192,6 +215,30 @@ true + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.19.1 + + + default-integration-test + + integration-test + verify + + + eu.dnetlib.dhp.common.IntegrationTest + + **/*Test.java + + + + + + + + org.apache.maven.plugins maven-javadoc-plugin @@ -205,7 +252,13 @@ maven-dependency-plugin 3.0.0 - + + + org.codehaus.mojo + build-helper-maven-plugin + 1.12 + + @@ -215,8 +268,42 @@ maven-release-plugin 2.5.3 + + org.jacoco + jacoco-maven-plugin + 0.7.9 + + + **/schemas/* + **/com/cloudera/**/* + **/org/apache/avro/io/**/* + + + + + default-prepare-agent + + prepare-agent + + + + default-report + prepare-package + + report + + + + + + + org.apache.maven.wagon + wagon-ssh + 2.10 + + @@ -250,9 +337,12 @@ UTF-8 UTF-8 + cdh5.9.2 2.6.0-${dhp.cdh.version} + 4.1.0-${dhp.cdh.version} 2.2.0 + 3.5 2.11.8