integrating the oozie workflow build/deploy/run mechanism, took inspiration from iis

This commit is contained in:
Claudio Atzori 2019-10-17 18:38:30 +02:00
parent bbb87d0e3d
commit 27db5afdad
25 changed files with 1975 additions and 4 deletions

View File

@ -0,0 +1,7 @@
Module utilized by `dhp-workflows`.
Contains all required resources by this parent module:
* assembly XML definitions
* build shell scripts
* oozie package commands for uploading, running and monitoring oozie workflows

View File

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-build</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>dhp-build-assembly-resources</artifactId>
<packaging>jar</packaging>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,32 @@
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>oozie-installer</id>
<formats>
<format>dir</format>
</formats>
<fileSets>
<fileSet>
<filtered>true</filtered>
<directory>${project.build.directory}/assembly-resources/commands</directory>
<!--
dziala dla (lokalnie zasoby modulu):
<directory>src/main/resources</directory>
nie dziala dla:
<directory>classpath:/commands</directory>
<directory>commands</directory>
<directory>classpath/src/main/resources</directory>
-->
<outputDirectory>/</outputDirectory>
<includes>
<include>**/*</include>
</includes>
<fileMode>0755</fileMode>
<lineEnding>unix</lineEnding>
</fileSet>
</fileSets>
<baseDirectory>/</baseDirectory>
</assembly>

View File

@ -0,0 +1,24 @@
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>tests</id>
<formats>
<format>jar</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>${project.build.testOutputDirectory}
</directory>
<outputDirectory />
</fileSet>
</fileSets>
<!-- <dependencySets>
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>lib</outputDirectory>
</dependencySet>
</dependencySets>-->
</assembly>

View File

@ -0,0 +1,3 @@
#!/bin/bash
hadoop fs -get ${workingDir}

View File

@ -0,0 +1,5 @@
#!/bin/bash
echo ""
echo "---->Contents of the working directory"
hadoop fs -ls ${workingDir}

View File

@ -0,0 +1,5 @@
Execute the scripts in the following order:
1. `upload_workflow.sh`
2. `run_workflow.sh`
3. `print_working_dir.sh` or `get_working_dir.sh`

View File

@ -0,0 +1,10 @@
#!/bin/bash
if [ $# = 0 ] ; then
oozie job -oozie ${oozieServiceLoc} -config job.properties -run
else
oozie job -oozie ${oozieServiceLoc} -config $1/job.properties -run
fi

View File

@ -0,0 +1,34 @@
#!/bin/bash
exec 3>&1
BASH_XTRACEFD=3
set -x ## print every executed command
if [ $# = 0 ] ; then
target_dir_root=`pwd`'/${oozieAppDir}'
else
target_dir_root=`readlink -f $1`'/${oozieAppDir}'
fi
# initial phase, creating symbolic links to jars in all subworkflows
# currently disabled
#libDir=$target_dir_root'/lib'
#dirs=`find $target_dir_root/* -maxdepth 10 -type d`
#for dir in $dirs
#do
# if [ -f $dir/workflow.xml ]
# then
# echo "creating symbolic links to jars in directory: $dir/lib"
# if [ ! -d "$dir/lib" ]; then
# mkdir $dir/lib
# fi
# find $libDir -type f -exec ln -s \{\} $dir/lib \;
# fi
#done
#uploading
hadoop fs -rm -r ${sandboxDir}
hadoop fs -mkdir -p ${sandboxDir}
hadoop fs -mkdir -p ${workingDir}
hadoop fs -put $target_dir_root ${sandboxDir}

View File

@ -0,0 +1,7 @@
#sandboxName when not provided explicitly will be generated
sandboxName=${sandboxName}
sandboxDir=/user/${iis.hadoop.frontend.user.name}/${sandboxName}
workingDir=${sandboxDir}/working_dir
oozie.wf.application.path = ${nameNode}${sandboxDir}/${oozieAppDir}
oozieTopWfApplicationPath = ${oozie.wf.application.path}

View File

@ -0,0 +1,6 @@
Maven plugin module utilized by `dhp-workflows` for proper `job.properties` file building.
It is based on http://site.kuali.org/maven/plugins/properties-maven-plugin/1.3.2/write-project-properties-mojo.html and supplemented with:
* handling includePropertyKeysFromFiles property allowing writing only properties listed in given property files
As a final outcome only properties listed in `<include>` element and listed as a keys in files from `<includePropertyKeysFromFiles>` element will be written to output file.

View File

@ -0,0 +1,81 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-build</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>dhp-build-properties-maven-plugin</artifactId>
<packaging>maven-plugin</packaging>
<dependencies>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>2.0</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-project</artifactId>
<version>2.0</version>
</dependency>
<dependency>
<groupId>org.kuali.maven.plugins</groupId>
<artifactId>properties-maven-plugin</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>annotations</artifactId>
<version>3.0.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<version>3.0.1</version>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<directory>target</directory>
<outputDirectory>target/classes</outputDirectory>
<finalName>${project.artifactId}-${project.version}</finalName>
<testOutputDirectory>target/test-classes</testOutputDirectory>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<phase>verify</phase>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<detectLinks>true</detectLinks>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,71 @@
package eu.dnetlib.maven.plugin.properties;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
/**
* Generates oozie properties which were not provided from commandline.
* @author mhorst
*
* @goal generate-properties
*/
public class GenerateOoziePropertiesMojo extends AbstractMojo {
public static final String PROPERTY_NAME_WF_SOURCE_DIR = "workflow.source.dir";
public static final String PROPERTY_NAME_SANDBOX_NAME = "sandboxName";
private final String[] limiters = {"iis", "dnetlib", "eu"};
@Override
public void execute() throws MojoExecutionException, MojoFailureException {
if (System.getProperties().containsKey(PROPERTY_NAME_WF_SOURCE_DIR) &&
!System.getProperties().containsKey(PROPERTY_NAME_SANDBOX_NAME)) {
String generatedSandboxName = generateSandboxName(System.getProperties().getProperty(
PROPERTY_NAME_WF_SOURCE_DIR));
if (generatedSandboxName!=null) {
System.getProperties().setProperty(PROPERTY_NAME_SANDBOX_NAME,
generatedSandboxName);
} else {
System.out.println("unable to generate sandbox name from path: " +
System.getProperties().getProperty(PROPERTY_NAME_WF_SOURCE_DIR));
}
}
}
/**
* Generates sandbox name from workflow source directory.
* @param wfSourceDir
* @return generated sandbox name
*/
private String generateSandboxName(String wfSourceDir) {
// utilize all dir names until finding one of the limiters
List<String> sandboxNameParts = new ArrayList<String>();
String[] tokens = StringUtils.split(wfSourceDir, File.separatorChar);
ArrayUtils.reverse(tokens);
if (tokens.length>0) {
for (String token : tokens) {
for (String limiter : limiters) {
if (limiter.equals(token)) {
return sandboxNameParts.size()>0?
StringUtils.join(sandboxNameParts.toArray()):null;
}
}
if (sandboxNameParts.size()>0) {
sandboxNameParts.add(0, File.separator);
}
sandboxNameParts.add(0, token);
}
return StringUtils.join(sandboxNameParts.toArray());
} else {
return null;
}
}
}

View File

@ -0,0 +1,436 @@
/**
*
* Licensed under the Educational Community License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.opensource.org/licenses/ecl2.php
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.dnetlib.maven.plugin.properties;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.project.MavenProject;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
/**
* Writes project properties for the keys listed in specified properties files.
* Based on:
* http://site.kuali.org/maven/plugins/properties-maven-plugin/1.3.2/write-project-properties-mojo.html
* @author mhorst
* @goal write-project-properties
*/
public class WritePredefinedProjectProperties extends AbstractMojo {
private static final String CR = "\r";
private static final String LF = "\n";
private static final String TAB = "\t";
protected static final String PROPERTY_PREFIX_ENV = "env.";
private static final String ENCODING_UTF8 = "utf8";
/**
* @parameter property="properties.includePropertyKeysFromFiles"
*/
private String[] includePropertyKeysFromFiles;
/**
* @parameter default-value="${project}"
* @required
* @readonly
*/
protected MavenProject project;
/**
* The file that properties will be written to
*
* @parameter property="properties.outputFile"
* default-value="${project.build.directory}/properties/project.properties";
* @required
*/
protected File outputFile;
/**
* If true, the plugin will silently ignore any non-existent properties files, and the build will continue
*
* @parameter property="properties.quiet" default-value="true"
*/
private boolean quiet;
/**
* Comma separated list of characters to escape when writing property values. cr=carriage return, lf=linefeed,
* tab=tab. Any other values are taken literally.
*
* @parameter default-value="cr,lf,tab" property="properties.escapeChars"
*/
private String escapeChars;
/**
* If true, the plugin will include system properties when writing the properties file. System properties override
* both environment variables and project properties.
*
* @parameter default-value="false" property="properties.includeSystemProperties"
*/
private boolean includeSystemProperties;
/**
* If true, the plugin will include environment variables when writing the properties file. Environment variables
* are prefixed with "env". Environment variables override project properties.
*
* @parameter default-value="false" property="properties.includeEnvironmentVariables"
*/
private boolean includeEnvironmentVariables;
/**
* Comma separated set of properties to exclude when writing the properties file
*
* @parameter property="properties.exclude"
*/
private String exclude;
/**
* Comma separated set of properties to write to the properties file. If provided, only the properties matching
* those supplied here will be written to the properties file.
*
* @parameter property="properties.include"
*/
private String include;
/* (non-Javadoc)
* @see org.apache.maven.plugin.AbstractMojo#execute()
*/
@Override
@SuppressFBWarnings({"NP_UNWRITTEN_FIELD","UWF_UNWRITTEN_FIELD"})
public void execute() throws MojoExecutionException, MojoFailureException {
Properties properties = new Properties();
// Add project properties
properties.putAll(project.getProperties());
if (includeEnvironmentVariables) {
// Add environment variables, overriding any existing properties with the same key
properties.putAll(getEnvironmentVariables());
}
if (includeSystemProperties) {
// Add system properties, overriding any existing properties with the same key
properties.putAll(System.getProperties());
}
// Remove properties as appropriate
trim(properties, exclude, include);
String comment = "# " + new Date() + "\n";
List<String> escapeTokens = getEscapeChars(escapeChars);
getLog().info("Creating " + outputFile);
writeProperties(outputFile, comment, properties, escapeTokens);
}
/**
* Provides environment variables.
* @return environment variables
*/
protected static Properties getEnvironmentVariables() {
Properties props = new Properties();
for (Entry<String, String> entry : System.getenv().entrySet()) {
props.setProperty(PROPERTY_PREFIX_ENV + entry.getKey(), entry.getValue());
}
return props;
}
/**
* Removes properties which should not be written.
* @param properties
* @param omitCSV
* @param includeCSV
* @throws MojoExecutionException
*/
protected void trim(Properties properties, String omitCSV, String includeCSV) throws MojoExecutionException {
List<String> omitKeys = getListFromCSV(omitCSV);
for (String key : omitKeys) {
properties.remove(key);
}
List<String> includeKeys = getListFromCSV(includeCSV);
// mh: including keys from predefined properties
if (includePropertyKeysFromFiles!=null && includePropertyKeysFromFiles.length>0) {
for (String currentIncludeLoc : includePropertyKeysFromFiles) {
if (validate(currentIncludeLoc)) {
Properties p = getProperties(currentIncludeLoc);
for (String key : p.stringPropertyNames()) {
includeKeys.add(key);
}
}
}
}
if (includeKeys!=null && !includeKeys.isEmpty()) {
// removing only when include keys provided
Set<String> keys = properties.stringPropertyNames();
for (String key : keys) {
if (!includeKeys.contains(key)) {
properties.remove(key);
}
}
}
}
/**
* Checks whether file exists.
* @param location
* @return true when exists, false otherwise.
*/
protected boolean exists(String location) {
if (StringUtils.isBlank(location)) {
return false;
}
File file = new File(location);
if (file.exists()) {
return true;
}
ResourceLoader loader = new DefaultResourceLoader();
Resource resource = loader.getResource(location);
return resource.exists();
}
/**
* Validates resource location.
* @param location
* @return true when valid, false otherwise
* @throws MojoExecutionException
*/
protected boolean validate(String location) throws MojoExecutionException {
boolean exists = exists(location);
if (exists) {
return true;
}
if (quiet) {
getLog().info("Ignoring non-existent properties file '" + location + "'");
return false;
} else {
throw new MojoExecutionException("Non-existent properties file '" + location + "'");
}
}
/**
* Provides input stream.
* @param location
* @return input stream
* @throws IOException
*/
protected InputStream getInputStream(String location) throws IOException {
File file = new File(location);
if (file.exists()) {
return new FileInputStream(location);
}
ResourceLoader loader = new DefaultResourceLoader();
Resource resource = loader.getResource(location);
return resource.getInputStream();
}
/**
* Creates properties for given location.
* @param location
* @return properties for given location
* @throws MojoExecutionException
*/
protected Properties getProperties(String location) throws MojoExecutionException {
InputStream in = null;
try {
Properties properties = new Properties();
in = getInputStream(location);
if (location.toLowerCase().endsWith(".xml")) {
properties.loadFromXML(in);
} else {
properties.load(in);
}
return properties;
} catch (IOException e) {
throw new MojoExecutionException("Error reading properties file " + location, e);
} finally {
IOUtils.closeQuietly(in);
}
}
/**
* Provides escape characters.
* @param escapeChars
* @return escape characters
*/
protected List<String> getEscapeChars(String escapeChars) {
List<String> tokens = getListFromCSV(escapeChars);
List<String> realTokens = new ArrayList<String>();
for (String token : tokens) {
String realToken = getRealToken(token);
realTokens.add(realToken);
}
return realTokens;
}
/**
* Provides real token.
* @param token
* @return real token
*/
protected String getRealToken(String token) {
if (token.equalsIgnoreCase("CR")) {
return CR;
} else if (token.equalsIgnoreCase("LF")) {
return LF;
} else if (token.equalsIgnoreCase("TAB")) {
return TAB;
} else {
return token;
}
}
/**
* Returns content.
* @param comment
* @param properties
* @param escapeTokens
* @return content
*/
protected String getContent(String comment, Properties properties, List<String> escapeTokens) {
List<String> names = new ArrayList<String>(properties.stringPropertyNames());
Collections.sort(names);
StringBuilder sb = new StringBuilder();
if (!StringUtils.isBlank(comment)) {
sb.append(comment);
}
for (String name : names) {
String value = properties.getProperty(name);
String escapedValue = escape(value, escapeTokens);
sb.append(name + "=" + escapedValue + "\n");
}
return sb.toString();
}
/**
* Writes properties to given file.
* @param file
* @param comment
* @param properties
* @param escapeTokens
* @throws MojoExecutionException
*/
protected void writeProperties(File file, String comment, Properties properties, List<String> escapeTokens)
throws MojoExecutionException {
try {
String content = getContent(comment, properties, escapeTokens);
FileUtils.writeStringToFile(file, content, ENCODING_UTF8);
} catch (IOException e) {
throw new MojoExecutionException("Error creating properties file", e);
}
}
/**
* Escapes characters.
* @param s
* @param escapeChars
* @return
*/
protected String escape(String s, List<String> escapeChars) {
String result = s;
for (String escapeChar : escapeChars) {
result = result.replace(escapeChar, getReplacementToken(escapeChar));
}
return result;
}
/**
* Provides replacement token.
* @param escapeChar
* @return replacement token
*/
protected String getReplacementToken(String escapeChar) {
if (escapeChar.equals(CR)) {
return "\\r";
} else if (escapeChar.equals(LF)) {
return "\\n";
} else if (escapeChar.equals(TAB)) {
return "\\t";
} else {
return "\\" + escapeChar;
}
}
/**
* Returns list from csv.
* @param csv
* @return list of values generated from CSV
*/
protected static final List<String> getListFromCSV(String csv) {
if (StringUtils.isBlank(csv)) {
return new ArrayList<String>();
}
List<String> list = new ArrayList<String>();
String[] tokens = StringUtils.split(csv, ",");
for (String token : tokens) {
list.add(token.trim());
}
return list;
}
public void setIncludeSystemProperties(boolean includeSystemProperties) {
this.includeSystemProperties = includeSystemProperties;
}
public void setEscapeChars(String escapeChars) {
this.escapeChars = escapeChars;
}
public void setIncludeEnvironmentVariables(boolean includeEnvironmentVariables) {
this.includeEnvironmentVariables = includeEnvironmentVariables;
}
public void setExclude(String exclude) {
this.exclude = exclude;
}
public void setInclude(String include) {
this.include = include;
}
public void setQuiet(boolean quiet) {
this.quiet = quiet;
}
/**
* Sets property files for which keys properties should be included.
* @param includePropertyKeysFromFiles
*/
public void setIncludePropertyKeysFromFiles(
String[] includePropertyKeysFromFiles) {
if (includePropertyKeysFromFiles!=null) {
this.includePropertyKeysFromFiles = Arrays.copyOf(
includePropertyKeysFromFiles,
includePropertyKeysFromFiles.length);
}
}
}

View File

@ -0,0 +1,101 @@
package eu.dnetlib.maven.plugin.properties;
import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_SANDBOX_NAME;
import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_WF_SOURCE_DIR;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import org.junit.Before;
import org.junit.Test;
/**
* @author mhorst
*
*/
public class GenerateOoziePropertiesMojoTest {
private GenerateOoziePropertiesMojo mojo = new GenerateOoziePropertiesMojo();
@Before
public void clearSystemProperties() {
System.clearProperty(PROPERTY_NAME_SANDBOX_NAME);
System.clearProperty(PROPERTY_NAME_WF_SOURCE_DIR);
}
@Test
public void testExecuteEmpty() throws Exception {
// execute
mojo.execute();
// assert
assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
}
@Test
public void testExecuteSandboxNameAlreadySet() throws Exception {
// given
String workflowSourceDir = "eu/dnetlib/iis/wf/transformers";
String sandboxName = "originalSandboxName";
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
System.setProperty(PROPERTY_NAME_SANDBOX_NAME, sandboxName);
// execute
mojo.execute();
// assert
assertEquals(sandboxName, System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
}
@Test
public void testExecuteEmptyWorkflowSourceDir() throws Exception {
// given
String workflowSourceDir = "";
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
// execute
mojo.execute();
// assert
assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
}
@Test
public void testExecuteNullSandboxNameGenerated() throws Exception {
// given
String workflowSourceDir = "eu/dnetlib/iis/";
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
// execute
mojo.execute();
// assert
assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
}
@Test
public void testExecute() throws Exception {
// given
String workflowSourceDir = "eu/dnetlib/iis/wf/transformers";
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
// execute
mojo.execute();
// assert
assertEquals("wf/transformers", System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
}
@Test
public void testExecuteWithoutRoot() throws Exception {
// given
String workflowSourceDir = "wf/transformers";
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
// execute
mojo.execute();
// assert
assertEquals("wf/transformers", System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
}
}

View File

@ -0,0 +1,365 @@
package eu.dnetlib.maven.plugin.properties;
import static eu.dnetlib.maven.plugin.properties.WritePredefinedProjectProperties.PROPERTY_PREFIX_ENV;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.doReturn;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Properties;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.project.MavenProject;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
/**
* @author mhorst
*
*/
@RunWith(MockitoJUnitRunner.class)
public class WritePredefinedProjectPropertiesTest {
@Rule
public TemporaryFolder testFolder = new TemporaryFolder();
@Mock
private MavenProject mavenProject;
private WritePredefinedProjectProperties mojo;
@Before
public void init() {
mojo = new WritePredefinedProjectProperties();
mojo.outputFile = getPropertiesFileLocation();
mojo.project = mavenProject;
doReturn(new Properties()).when(mavenProject).getProperties();
}
// ----------------------------------- TESTS ---------------------------------------------
@Test
public void testExecuteEmpty() throws Exception {
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertEquals(0, storedProperties.size());
}
@Test
public void testExecuteWithProjectProperties() throws Exception {
// given
String key = "projectPropertyKey";
String value = "projectPropertyValue";
Properties projectProperties = new Properties();
projectProperties.setProperty(key, value);
doReturn(projectProperties).when(mavenProject).getProperties();
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertEquals(1, storedProperties.size());
assertTrue(storedProperties.containsKey(key));
assertEquals(value, storedProperties.getProperty(key));
}
@Test(expected=MojoExecutionException.class)
public void testExecuteWithProjectPropertiesAndInvalidOutputFile() throws Exception {
// given
String key = "projectPropertyKey";
String value = "projectPropertyValue";
Properties projectProperties = new Properties();
projectProperties.setProperty(key, value);
doReturn(projectProperties).when(mavenProject).getProperties();
mojo.outputFile = testFolder.getRoot();
// execute
mojo.execute();
}
@Test
public void testExecuteWithProjectPropertiesExclusion() throws Exception {
// given
String key = "projectPropertyKey";
String value = "projectPropertyValue";
String excludedKey = "excludedPropertyKey";
String excludedValue = "excludedPropertyValue";
Properties projectProperties = new Properties();
projectProperties.setProperty(key, value);
projectProperties.setProperty(excludedKey, excludedValue);
doReturn(projectProperties).when(mavenProject).getProperties();
mojo.setExclude(excludedKey);
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertEquals(1, storedProperties.size());
assertTrue(storedProperties.containsKey(key));
assertEquals(value, storedProperties.getProperty(key));
}
@Test
public void testExecuteWithProjectPropertiesInclusion() throws Exception {
// given
String key = "projectPropertyKey";
String value = "projectPropertyValue";
String includedKey = "includedPropertyKey";
String includedValue = "includedPropertyValue";
Properties projectProperties = new Properties();
projectProperties.setProperty(key, value);
projectProperties.setProperty(includedKey, includedValue);
doReturn(projectProperties).when(mavenProject).getProperties();
mojo.setInclude(includedKey);
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertEquals(1, storedProperties.size());
assertTrue(storedProperties.containsKey(includedKey));
assertEquals(includedValue, storedProperties.getProperty(includedKey));
}
@Test
public void testExecuteIncludingPropertyKeysFromFile() throws Exception {
// given
String key = "projectPropertyKey";
String value = "projectPropertyValue";
String includedKey = "includedPropertyKey";
String includedValue = "includedPropertyValue";
Properties projectProperties = new Properties();
projectProperties.setProperty(key, value);
projectProperties.setProperty(includedKey, includedValue);
doReturn(projectProperties).when(mavenProject).getProperties();
File includedPropertiesFile = new File(testFolder.getRoot(), "included.properties");
Properties includedProperties = new Properties();
includedProperties.setProperty(includedKey, "irrelevantValue");
includedProperties.store(new FileWriter(includedPropertiesFile), null);
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertEquals(1, storedProperties.size());
assertTrue(storedProperties.containsKey(includedKey));
assertEquals(includedValue, storedProperties.getProperty(includedKey));
}
@Test
public void testExecuteIncludingPropertyKeysFromClasspathResource() throws Exception {
// given
String key = "projectPropertyKey";
String value = "projectPropertyValue";
String includedKey = "includedPropertyKey";
String includedValue = "includedPropertyValue";
Properties projectProperties = new Properties();
projectProperties.setProperty(key, value);
projectProperties.setProperty(includedKey, includedValue);
doReturn(projectProperties).when(mavenProject).getProperties();
mojo.setIncludePropertyKeysFromFiles(new String[] {"/eu/dnetlib/maven/plugin/properties/included.properties"});
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertEquals(1, storedProperties.size());
assertTrue(storedProperties.containsKey(includedKey));
assertEquals(includedValue, storedProperties.getProperty(includedKey));
}
@Test(expected=MojoExecutionException.class)
public void testExecuteIncludingPropertyKeysFromBlankLocation() throws Exception {
// given
String key = "projectPropertyKey";
String value = "projectPropertyValue";
String includedKey = "includedPropertyKey";
String includedValue = "includedPropertyValue";
Properties projectProperties = new Properties();
projectProperties.setProperty(key, value);
projectProperties.setProperty(includedKey, includedValue);
doReturn(projectProperties).when(mavenProject).getProperties();
mojo.setIncludePropertyKeysFromFiles(new String[] {""});
// execute
mojo.execute();
}
@Test
public void testExecuteIncludingPropertyKeysFromXmlFile() throws Exception {
// given
String key = "projectPropertyKey";
String value = "projectPropertyValue";
String includedKey = "includedPropertyKey";
String includedValue = "includedPropertyValue";
Properties projectProperties = new Properties();
projectProperties.setProperty(key, value);
projectProperties.setProperty(includedKey, includedValue);
doReturn(projectProperties).when(mavenProject).getProperties();
File includedPropertiesFile = new File(testFolder.getRoot(), "included.xml");
Properties includedProperties = new Properties();
includedProperties.setProperty(includedKey, "irrelevantValue");
includedProperties.storeToXML(new FileOutputStream(includedPropertiesFile), null);
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertEquals(1, storedProperties.size());
assertTrue(storedProperties.containsKey(includedKey));
assertEquals(includedValue, storedProperties.getProperty(includedKey));
}
@Test(expected=MojoExecutionException.class)
public void testExecuteIncludingPropertyKeysFromInvalidXmlFile() throws Exception {
// given
String key = "projectPropertyKey";
String value = "projectPropertyValue";
String includedKey = "includedPropertyKey";
String includedValue = "includedPropertyValue";
Properties projectProperties = new Properties();
projectProperties.setProperty(key, value);
projectProperties.setProperty(includedKey, includedValue);
doReturn(projectProperties).when(mavenProject).getProperties();
File includedPropertiesFile = new File(testFolder.getRoot(), "included.xml");
Properties includedProperties = new Properties();
includedProperties.setProperty(includedKey, "irrelevantValue");
includedProperties.store(new FileOutputStream(includedPropertiesFile), null);
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
// execute
mojo.execute();
}
@Test
public void testExecuteWithQuietModeOn() throws Exception {
// given
mojo.setQuiet(true);
mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"});
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertEquals(0, storedProperties.size());
}
@Test(expected=MojoExecutionException.class)
public void testExecuteIncludingPropertyKeysFromInvalidFile() throws Exception {
// given
mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"});
// execute
mojo.execute();
}
@Test
public void testExecuteWithEnvironmentProperties() throws Exception {
// given
mojo.setIncludeEnvironmentVariables(true);
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertTrue(storedProperties.size() > 0);
for (Object currentKey : storedProperties.keySet()) {
assertTrue(((String)currentKey).startsWith(PROPERTY_PREFIX_ENV));
}
}
@Test
public void testExecuteWithSystemProperties() throws Exception {
// given
String key = "systemPropertyKey";
String value = "systemPropertyValue";
System.setProperty(key, value);
mojo.setIncludeSystemProperties(true);
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertTrue(storedProperties.size() > 0);
assertTrue(storedProperties.containsKey(key));
assertEquals(value, storedProperties.getProperty(key));
}
@Test
public void testExecuteWithSystemPropertiesAndEscapeChars() throws Exception {
// given
String key = "systemPropertyKey ";
String value = "systemPropertyValue";
System.setProperty(key, value);
mojo.setIncludeSystemProperties(true);
String escapeChars = "cr,lf,tab,|";
mojo.setEscapeChars(escapeChars);
// execute
mojo.execute();
// assert
assertTrue(mojo.outputFile.exists());
Properties storedProperties = getStoredProperties();
assertTrue(storedProperties.size() > 0);
assertFalse(storedProperties.containsKey(key));
assertTrue(storedProperties.containsKey(key.trim()));
assertEquals(value, storedProperties.getProperty(key.trim()));
}
// ----------------------------------- PRIVATE -------------------------------------------
private File getPropertiesFileLocation() {
return new File(testFolder.getRoot(), "test.properties");
}
private Properties getStoredProperties() throws FileNotFoundException, IOException {
Properties properties = new Properties();
properties.load(new FileInputStream(getPropertiesFileLocation()));
return properties;
}
}

View File

@ -9,7 +9,8 @@
<artifactId>dhp-build</artifactId> <artifactId>dhp-build</artifactId>
<packaging>pom</packaging> <packaging>pom</packaging>
<modules> <modules>
<module>dhp-build-assembly-resources</module>
<module>dhp-build-properties-maven-plugin</module>
</modules> </modules>
</project> </project>

View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dhp-workflows</artifactId>
<groupId>eu.dnetlib.dhp</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dhp-distcp</artifactId>
</project>

View File

@ -0,0 +1,18 @@
<configuration>
<property>
<name>jobTracker</name>
<value>yarnRM</value>
</property>
<property>
<name>nameNode</name>
<value>hdfs://nameservice1</value>
</property>
<property>
<name>sourceNN</name>
<value>webhdfs://namenode2.hadoop.dm.openaire.eu:50071</value>
</property>
<property>
<name>oozie.use.system.libpath</name>
<value>true</value>
</property>
</configuration>

View File

@ -0,0 +1,46 @@
<workflow-app name="distcp" xmlns="uri:oozie:workflow:0.5">
<parameters>
<property>
<name>sourceNN</name>
<description>the source name node</description>
</property>
<property>
<name>sourcePath</name>
<description>the source path</description>
</property>
<property>
<name>targetPath</name>
<description>the target path</description>
</property>
<property>
<name>hbase_dump_distcp_memory_mb</name>
<value>6144</value>
<description>memory for distcp action copying InfoSpace dump from remote cluster</description>
</property>
<property>
<name>hbase_dump_distcp_num_maps</name>
<value>1</value>
<description>maximum number of simultaneous copies of InfoSpace dump from remote location</description>
</property>
</parameters>
<start to="distcp"/>
<kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
</kill>
<action name="distcp">
<distcp xmlns="uri:oozie:distcp-action:0.2">
<arg>-Dmapreduce.map.memory.mb=${hbase_dump_distcp_memory_mb}</arg>
<arg>-pb</arg>
<arg>-m ${hbase_dump_distcp_num_maps}</arg>
<arg>${sourceNN}/${sourcePath}</arg>
<arg>${nameNode}/${targetPath}</arg>
</distcp>
<ok to="End" />
<error to="Kill" />
</action>
<end name="End"/>
</workflow-app>

View File

@ -1,16 +1,604 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<groupId>eu.dnetlib.dhp</groupId> <groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp</artifactId> <artifactId>dhp</artifactId>
<version>1.0.0-SNAPSHOT</version> <version>1.0.0-SNAPSHOT</version>
<relativePath>../</relativePath> <relativePath>../</relativePath>
</parent> </parent>
<artifactId>dhp-workflows</artifactId> <artifactId>dhp-workflows</artifactId>
<packaging>pom</packaging> <packaging>pom</packaging>
<modules> <modules>
<module>dhp-aggregation</module> <module>dhp-aggregation</module>
<module>dhp-distcp</module>
</modules> </modules>
<pluginRepositories>
<pluginRepository>
<id>iis-releases</id>
<name>iis releases plugin repository</name>
<url>http://maven.ceon.pl/artifactory/iis-releases</url>
<layout>default</layout>
</pluginRepository>
</pluginRepositories>
<properties>
<maven.build.timestamp.format>yyyy-MM-dd_HH_mm</maven.build.timestamp.format>
<!-- default Oozie installer properties requred to be defined at pom.xml level -->
<!-- other project properties are defined in project-default.properties -->
<oozie.package.file.name>oozie-package</oozie.package.file.name>
<!-- notice: sandboxName is generated based on workflow.source.dir property -->
<workflow.source.dir>src/test/resources/define/path/pointing/to/directory/holding/oozie_app</workflow.source.dir>
<oozieAppDir>oozie_app</oozieAppDir>
<queueName>default</queueName>
<importerQueueName>default</importerQueueName>
<oozieLauncherQueueName>default</oozieLauncherQueueName>
<primed.dir>primed</primed.dir>
<oozie.package.dependencies.include.scope>runtime</oozie.package.dependencies.include.scope>
<oozie.package.dependencies.exclude.scope></oozie.package.dependencies.exclude.scope>
<oozie.package.skip.test.jar>true</oozie.package.skip.test.jar>
<dhpConnectionProperties>${user.home}/.dhp/application.properties</dhpConnectionProperties>
<output.dir.name>${maven.build.timestamp}</output.dir.name>
<projectVersion>${project.version}</projectVersion>
<oozie.use.system.libpath>true</oozie.use.system.libpath>
</properties>
<dependencies>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-build-assembly-resources</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-client</artifactId>
</dependency>
<dependency>
<groupId>net.schmizz</groupId>
<artifactId>sshj</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>
<profile>
<!-- This profile sets properties that are required for test oozie workflows To be used only with 'oozie-package' profile -->
<id>attach-test-resources</id>
<properties>
<!--overriding default scope (set to 'runtime') with the 'test' value. Test resources attached to oozie package requires all test dependencies. -->
<oozie.package.dependencies.include.scope></oozie.package.dependencies.include.scope>
<oozie.package.dependencies.exclude.scope>provided</oozie.package.dependencies.exclude.scope>
<!-- Do not skip creation of test jar for priming (in oozie-package profile) -->
<oozie.package.skip.test.jar>false</oozie.package.skip.test.jar>
</properties>
</profile>
<profile>
<id>oozie-package</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.4.1</version>
<executions>
<execution>
<id>enforce-connection-properties-file-existence</id>
<phase>initialize</phase>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireFilesExist>
<files>
<file>${dhpConnectionProperties}</file>
</files>
<message>
The file with connection properties could not be found. Please, create the ${dhpConnectionProperties} file or set the location to another already created file by using
-DdhpConnectionProperties property.
</message>
</requireFilesExist>
</rules>
<fail>true</fail>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy dependencies</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeScope>${oozie.package.dependencies.include.scope}</includeScope>
<excludeScope>${oozie.package.dependencies.exclude.scope}</excludeScope>
<silent>true</silent>
</configuration>
</execution>
</executions>
</plugin>
<!-- Plugin originally defined in attach-test-resources It was moved here to ensure that it will execute before priming -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<id>attach-test-resources-package</id>
<phase>prepare-package</phase>
<goals>
<goal>test-jar</goal>
</goals>
<configuration>
<skip>${oozie.package.skip.test.jar}</skip>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>eu.dnetlib.primer</groupId>
<artifactId>primer-maven-plugin</artifactId>
<executions>
<execution>
<id>priming</id>
<phase>prepare-package</phase>
<goals>
<goal>prime</goal>
</goals>
<configuration>
<classProviderFiles>
<classProviderFile>${project.build.directory}/dependency/*.jar</classProviderFile>
<classProviderFile>${project.build.directory}/*-tests.jar</classProviderFile>
<classProviderFile>${project.build.directory}/classes</classProviderFile>
</classProviderFiles>
<coansysPackageDir>${project.build.directory}/dependency</coansysPackageDir>
<destination>${project.build.directory}/${primed.dir}</destination>
<classpath>${workflow.source.dir}</classpath>
</configuration>
</execution>
</executions>
</plugin>
<!-- reading job.properties to use them in .sh scripts -->
<plugin>
<groupId>org.kuali.maven.plugins</groupId>
<artifactId>properties-maven-plugin</artifactId>
<version>1.3.2</version>
<dependencies>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-build-assembly-resources</artifactId>
<version>${project.version}</version>
<!-- contains project-default.properties -->
</dependency>
</dependencies>
<executions>
<execution>
<id>reading-dhp-properties</id>
<phase>initialize</phase>
<goals>
<goal>read-project-properties</goal>
</goals>
<configuration>
<locations>
<location>${dhpConnectionProperties}</location>
</locations>
<quiet>false</quiet>
</configuration>
</execution>
<execution>
<id>read-default-properties</id>
<phase>prepare-package</phase>
<goals>
<goal>read-project-properties</goal>
</goals>
<configuration>
<locations>
<location>classpath:project-default.properties</location>
</locations>
<quiet>true</quiet>
</configuration>
</execution>
<execution>
<id>read-job-properties</id>
<phase>prepare-package</phase>
<goals>
<goal>read-project-properties</goal>
</goals>
<configuration>
<locations>
<param>${project.build.directory}/${primed.dir}/job.properties</param>
<param>job-override.properties</param>
</locations>
<quiet>true</quiet>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-build-properties-maven-plugin</artifactId>
<executions>
<execution>
<phase>validate</phase>
<goals>
<goal>generate-properties</goal>
<!-- generates sandboxName based on workflow.source.dir when not specified as commandline parameter -->
</goals>
<configuration>
</configuration>
</execution>
<execution>
<id>write-job-properties</id>
<phase>prepare-package</phase>
<goals>
<goal>write-project-properties</goal>
</goals>
<configuration>
<outputFile>target/${oozie.package.file.name}/job.properties</outputFile>
<!-- notice: dots are not allowed for job.properties! -->
<include>nameNode,jobTracker,queueName,importerQueueName,oozieLauncherQueueName,
workingDir,oozieTopWfApplicationPath,oozieServiceLoc,
sparkDriverMemory,sparkExecutorMemory,sparkExecutorCores,
oozie.wf.application.path,projectVersion,oozie.use.system.libpath</include>
<includeSystemProperties>true</includeSystemProperties>
<includePropertyKeysFromFiles>
<!-- <param>${workflow.source.dir}/job.properties</param> -->
<param>${project.build.directory}/${primed.dir}/job.properties</param>
<param>job-override.properties</param>
</includePropertyKeysFromFiles>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>2.1.11</version>
<executions>
<execution>
<goals>
<goal>revision</goal>
</goals>
</execution>
</executions>
<configuration>
<verbose>true</verbose>
<dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<generateGitPropertiesFilename>target/${oozie.package.file.name}/${oozieAppDir}/version.properties</generateGitPropertiesFilename>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.0.0</version>
<dependencies>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-build-assembly-resources</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<id>assembly-oozie-installer</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<finalName>${oozie.package.file.name}_shell_scripts</finalName>
<descriptorRefs>
<descriptorRef>oozie-installer</descriptorRef>
</descriptorRefs>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<!-- this plugin prepares oozie installer package-->
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<!-- extracting shared resources phase -->
<execution>
<id>installer-copy-custom</id>
<phase>process-resources</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<property name="assembly-resources.loc" value="${maven.dependency.eu.dnetlib.dhp.dhp-build-assembly-resources.jar.path}" />
<unjar src="${assembly-resources.loc}" dest="${project.build.directory}/assembly-resources" />
</tasks>
</configuration>
</execution>
<!-- packaging phase -->
<execution>
<phase>package</phase>
<configuration>
<tasks>
<!-- copying workflow resources -->
<mkdir dir="target/${oozie.package.file.name}" />
<mkdir dir="target/${oozie.package.file.name}/${oozieAppDir}" />
<copy todir="target/${oozie.package.file.name}/${oozieAppDir}">
<!-- <fileset dir="${workflow.source.dir}/${oozieAppDir}" /> replacing with primed dir location -->
<fileset dir="target/${primed.dir}/${oozieAppDir}" />
</copy>
<!-- copying all jars to oozie lib directory -->
<mkdir dir="target/${oozie.package.file.name}/${oozieAppDir}/lib" />
<copy todir="target/${oozie.package.file.name}/${oozieAppDir}/lib">
<fileset dir="${project.build.directory}/dependency" />
</copy>
<!-- copying current module lib -->
<copy todir="target/${oozie.package.file.name}/${oozieAppDir}/lib">
<fileset dir="${project.build.directory}">
<include name="*.jar" />
</fileset>
</copy>
<fixcrlf srcdir="target/${oozie.package.file.name}/${oozieAppDir}/" encoding="UTF-8" outputencoding="UTF-8" includes="**/*.sh,**/*.json,**/*.py,**/*.sql" eol="lf"/>
<!-- creating tar.gz package -->
<tar destfile="target/${oozie.package.file.name}.tar.gz" compression="gzip">
<tarfileset dir="target/${oozie.package.file.name}" />
<tarfileset dir="target/${oozie.package.file.name}_shell_scripts" filemode="0755">
<include name="**/*.sh" />
</tarfileset>
<tarfileset dir="target/${oozie.package.file.name}_shell_scripts" filemode="0644">
<exclude name="**/*.sh" />
</tarfileset>
</tar>
<!-- cleanup -->
<delete dir="target/${oozie.package.file.name}" />
<delete dir="target/${oozie.package.file.name}_shell_scripts" />
</tasks>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>deploy</id>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.5.0</version>
<executions>
<execution>
<id>create-target-dir</id>
<phase>package</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>ssh</executable>
<arguments>
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
<argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
<argument>-o StrictHostKeyChecking=no</argument>
<argument>rm -rf ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; mkdir -p ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>upload-oozie-package</id>
<phase>package</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>scp</executable>
<arguments>
<argument>-P ${dhp.hadoop.frontend.port.ssh}</argument>
<argument>-o StrictHostKeyChecking=no</argument>
<argument>target/${oozie.package.file.name}.tar.gz</argument>
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}:${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/${oozie.package.file.name}.tar.gz</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>extract-and-upload-to-hdfs</id>
<phase>package</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>ssh</executable>
<!-- <outputFile>target/redirected_upload.log</outputFile> -->
<arguments>
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
<argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
<argument>-o StrictHostKeyChecking=no</argument>
<argument>cd ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; </argument>
<argument>tar -zxf oozie-package.tar.gz; </argument>
<argument>rm ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/oozie-package.tar.gz; </argument>
<argument>./upload_workflow.sh</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>run</id>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.5.0</version>
<executions>
<execution>
<id>run-job</id>
<phase>package</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>ssh</executable>
<!-- this file will be used by test verification profile reading job identifier -->
<outputFile>${oozie.execution.log.file.location}</outputFile>
<arguments>
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
<argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
<argument>-o StrictHostKeyChecking=no</argument>
<argument>cd ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; </argument>
<argument>./run_workflow.sh</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>show-run-log-on-stdout</id>
<phase>package</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>cat</executable>
<arguments>
<argument>${oozie.execution.log.file.location}</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<!-- this profile is handling unit and integration test definitions of all child modules -->
<id>child-tests</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<id>integration-test-package</id>
<phase>integration-test</phase>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<excludedGroups>eu.dnetlib.iis.common.IntegrationTest</excludedGroups>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
<id>default-integration-test</id>
<configuration>
<argLine>-Xmx1024m</argLine>
<systemPropertiesVariables>
<!-- if dhpConnectionProperties is not defined, then ${user.home}/.dhp/integration-test.properties will be used -->
<dhpConnectionProperties>${dhpConnectionProperties}</dhpConnectionProperties>
<output.dir.name>${output.dir.name}</output.dir.name>
</systemPropertiesVariables>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<activation>
<file>
<missing>src/main/resources/parent.marker</missing>
</file>
</activation>
</profile>
</profiles>
<build>
<pluginManagement>
<plugins>
<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself. -->
<plugin>
<groupId>org.eclipse.m2e</groupId>
<artifactId>lifecycle-mapping</artifactId>
<version>1.0.0</version>
<configuration>
<lifecycleMappingMetadata>
<pluginExecutions>
<pluginExecution>
<pluginExecutionFilter>
<groupId>
org.kuali.maven.plugins
</groupId>
<artifactId>
properties-maven-plugin
</artifactId>
<versionRange>
[1.3.2,)
</versionRange>
<goals>
<goal>
read-project-properties
</goal>
<goal>
write-project-properties
</goal>
</goals>
</pluginExecutionFilter>
<action>
<ignore />
</action>
</pluginExecution>
<!-- copy-dependency plugin -->
<pluginExecution>
<pluginExecutionFilter>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<versionRange>[1.0.0,)</versionRange>
<goals>
<goal>copy-dependencies</goal>
</goals>
</pluginExecutionFilter>
<action>
<ignore />
</action>
</pluginExecution>
</pluginExecutions>
</lifecycleMappingMetadata>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>
</project> </project>

94
pom.xml
View File

@ -20,7 +20,7 @@
</licenses> </licenses>
<modules> <modules>
<!--<module>dhp-build</module>--> <module>dhp-build</module>
<module>dhp-common</module> <module>dhp-common</module>
<module>dhp-workflows</module> <module>dhp-workflows</module>
<module>dhp-applications</module> <module>dhp-applications</module>
@ -60,6 +60,8 @@
<enabled>true</enabled> <enabled>true</enabled>
</releases> </releases>
</repository> </repository>
<!--
<repository> <repository>
<id>dnet45-bootstrap-release</id> <id>dnet45-bootstrap-release</id>
<name>dnet45 bootstrap release</name> <name>dnet45 bootstrap release</name>
@ -72,6 +74,7 @@
<enabled>true</enabled> <enabled>true</enabled>
</releases> </releases>
</repository> </repository>
-->
<repository> <repository>
<id>cloudera</id> <id>cloudera</id>
@ -142,6 +145,26 @@
<artifactId>commons-cli</artifactId> <artifactId>commons-cli</artifactId>
<version>1.4</version> <version>1.4</version>
</dependency> </dependency>
<dependency>
<groupId>net.schmizz</groupId>
<artifactId>sshj</artifactId>
<version>0.10.0</version>
</dependency>
<dependency>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-client</artifactId>
<version>${dhp.oozie.version}</version>
<exclusions>
<!-- conflicts -->
<exclusion>
<artifactId>slf4j-simple</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
@ -192,6 +215,30 @@
<redirectTestOutputToFile>true</redirectTestOutputToFile> <redirectTestOutputToFile>true</redirectTestOutputToFile>
</configuration> </configuration>
</plugin> </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.19.1</version>
<executions>
<execution>
<id>default-integration-test</id>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
<configuration>
<groups>eu.dnetlib.dhp.common.IntegrationTest</groups>
<includes>
<include>**/*Test.java</include>
</includes>
</configuration>
</execution>
</executions>
</plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId> <artifactId>maven-javadoc-plugin</artifactId>
@ -205,7 +252,13 @@
<artifactId>maven-dependency-plugin</artifactId> <artifactId>maven-dependency-plugin</artifactId>
<version>3.0.0</version> <version>3.0.0</version>
</plugin> </plugin>
<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself. -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.12</version>
</plugin>
</plugins> </plugins>
</pluginManagement> </pluginManagement>
@ -215,8 +268,42 @@
<artifactId>maven-release-plugin</artifactId> <artifactId>maven-release-plugin</artifactId>
<version>2.5.3</version> <version>2.5.3</version>
</plugin> </plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.7.9</version>
<configuration>
<excludes>
<exclude>**/schemas/*</exclude>
<exclude>**/com/cloudera/**/*</exclude>
<exclude>**/org/apache/avro/io/**/*</exclude>
</excludes>
</configuration>
<executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>default-report</id>
<phase>prepare-package</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins> </plugins>
<extensions>
<extension>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-ssh</artifactId>
<version>2.10</version>
</extension>
</extensions>
</build> </build>
@ -250,9 +337,12 @@
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<dhp.cdh.version>cdh5.9.2</dhp.cdh.version> <dhp.cdh.version>cdh5.9.2</dhp.cdh.version>
<dhp.hadoop.version>2.6.0-${dhp.cdh.version}</dhp.hadoop.version> <dhp.hadoop.version>2.6.0-${dhp.cdh.version}</dhp.hadoop.version>
<dhp.oozie.version>4.1.0-${dhp.cdh.version}</dhp.oozie.version>
<dhp.spark.version>2.2.0</dhp.spark.version> <dhp.spark.version>2.2.0</dhp.spark.version>
<dhp.commons.lang.version>3.5</dhp.commons.lang.version> <dhp.commons.lang.version>3.5</dhp.commons.lang.version>
<scala.version>2.11.8</scala.version> <scala.version>2.11.8</scala.version>
</properties> </properties>