forked from D-Net/dnet-hadoop
integrating the oozie workflow build/deploy/run mechanism, took inspiration from iis
This commit is contained in:
parent
bbb87d0e3d
commit
27db5afdad
|
|
@ -0,0 +1,7 @@
|
|||
Module utilized by `dhp-workflows`.
|
||||
|
||||
Contains all required resources by this parent module:
|
||||
|
||||
* assembly XML definitions
|
||||
* build shell scripts
|
||||
* oozie package commands for uploading, running and monitoring oozie workflows
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-build</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>dhp-build-assembly-resources</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
<assembly
|
||||
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||
|
||||
<id>oozie-installer</id>
|
||||
<formats>
|
||||
<format>dir</format>
|
||||
</formats>
|
||||
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<filtered>true</filtered>
|
||||
<directory>${project.build.directory}/assembly-resources/commands</directory>
|
||||
<!--
|
||||
dziala dla (lokalnie zasoby modulu):
|
||||
<directory>src/main/resources</directory>
|
||||
nie dziala dla:
|
||||
<directory>classpath:/commands</directory>
|
||||
<directory>commands</directory>
|
||||
<directory>classpath/src/main/resources</directory>
|
||||
-->
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<includes>
|
||||
<include>**/*</include>
|
||||
</includes>
|
||||
<fileMode>0755</fileMode>
|
||||
<lineEnding>unix</lineEnding>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
<baseDirectory>/</baseDirectory>
|
||||
</assembly>
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
<assembly
|
||||
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||
|
||||
<id>tests</id>
|
||||
<formats>
|
||||
<format>jar</format>
|
||||
</formats>
|
||||
<includeBaseDirectory>false</includeBaseDirectory>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>${project.build.testOutputDirectory}
|
||||
</directory>
|
||||
<outputDirectory />
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
<!-- <dependencySets>
|
||||
<dependencySet>
|
||||
<useProjectArtifact>false</useProjectArtifact>
|
||||
<outputDirectory>lib</outputDirectory>
|
||||
</dependencySet>
|
||||
</dependencySets>-->
|
||||
</assembly>
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
#!/bin/bash
|
||||
hadoop fs -get ${workingDir}
|
||||
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
echo ""
|
||||
echo "---->Contents of the working directory"
|
||||
hadoop fs -ls ${workingDir}
|
||||
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
Execute the scripts in the following order:
|
||||
|
||||
1. `upload_workflow.sh`
|
||||
2. `run_workflow.sh`
|
||||
3. `print_working_dir.sh` or `get_working_dir.sh`
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ $# = 0 ] ; then
|
||||
oozie job -oozie ${oozieServiceLoc} -config job.properties -run
|
||||
else
|
||||
oozie job -oozie ${oozieServiceLoc} -config $1/job.properties -run
|
||||
fi
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
#!/bin/bash
|
||||
exec 3>&1
|
||||
BASH_XTRACEFD=3
|
||||
set -x ## print every executed command
|
||||
|
||||
|
||||
if [ $# = 0 ] ; then
|
||||
target_dir_root=`pwd`'/${oozieAppDir}'
|
||||
else
|
||||
target_dir_root=`readlink -f $1`'/${oozieAppDir}'
|
||||
fi
|
||||
|
||||
# initial phase, creating symbolic links to jars in all subworkflows
|
||||
# currently disabled
|
||||
#libDir=$target_dir_root'/lib'
|
||||
#dirs=`find $target_dir_root/* -maxdepth 10 -type d`
|
||||
#for dir in $dirs
|
||||
#do
|
||||
# if [ -f $dir/workflow.xml ]
|
||||
# then
|
||||
# echo "creating symbolic links to jars in directory: $dir/lib"
|
||||
# if [ ! -d "$dir/lib" ]; then
|
||||
# mkdir $dir/lib
|
||||
# fi
|
||||
# find $libDir -type f -exec ln -s \{\} $dir/lib \;
|
||||
# fi
|
||||
#done
|
||||
|
||||
|
||||
#uploading
|
||||
hadoop fs -rm -r ${sandboxDir}
|
||||
hadoop fs -mkdir -p ${sandboxDir}
|
||||
hadoop fs -mkdir -p ${workingDir}
|
||||
hadoop fs -put $target_dir_root ${sandboxDir}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#sandboxName when not provided explicitly will be generated
|
||||
sandboxName=${sandboxName}
|
||||
sandboxDir=/user/${iis.hadoop.frontend.user.name}/${sandboxName}
|
||||
workingDir=${sandboxDir}/working_dir
|
||||
oozie.wf.application.path = ${nameNode}${sandboxDir}/${oozieAppDir}
|
||||
oozieTopWfApplicationPath = ${oozie.wf.application.path}
|
||||
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
Maven plugin module utilized by `dhp-workflows` for proper `job.properties` file building.
|
||||
|
||||
It is based on http://site.kuali.org/maven/plugins/properties-maven-plugin/1.3.2/write-project-properties-mojo.html and supplemented with:
|
||||
|
||||
* handling includePropertyKeysFromFiles property allowing writing only properties listed in given property files
|
||||
As a final outcome only properties listed in `<include>` element and listed as a keys in files from `<includePropertyKeysFromFiles>` element will be written to output file.
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-build</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>dhp-build-properties-maven-plugin</artifactId>
|
||||
<packaging>maven-plugin</packaging>
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.maven</groupId>
|
||||
<artifactId>maven-plugin-api</artifactId>
|
||||
<version>2.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.maven</groupId>
|
||||
<artifactId>maven-project</artifactId>
|
||||
<version>2.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.kuali.maven.plugins</groupId>
|
||||
<artifactId>properties-maven-plugin</artifactId>
|
||||
<version>1.3.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>annotations</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>jsr305</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
<build>
|
||||
<directory>target</directory>
|
||||
<outputDirectory>target/classes</outputDirectory>
|
||||
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||
<testOutputDirectory>target/test-classes</testOutputDirectory>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-source-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-sources</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>jar-no-fork</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<configuration>
|
||||
<detectLinks>true</detectLinks>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
package eu.dnetlib.maven.plugin.properties;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.maven.plugin.AbstractMojo;
|
||||
import org.apache.maven.plugin.MojoExecutionException;
|
||||
import org.apache.maven.plugin.MojoFailureException;
|
||||
|
||||
/**
|
||||
* Generates oozie properties which were not provided from commandline.
|
||||
* @author mhorst
|
||||
*
|
||||
* @goal generate-properties
|
||||
*/
|
||||
public class GenerateOoziePropertiesMojo extends AbstractMojo {
|
||||
|
||||
public static final String PROPERTY_NAME_WF_SOURCE_DIR = "workflow.source.dir";
|
||||
public static final String PROPERTY_NAME_SANDBOX_NAME = "sandboxName";
|
||||
|
||||
private final String[] limiters = {"iis", "dnetlib", "eu"};
|
||||
|
||||
@Override
|
||||
public void execute() throws MojoExecutionException, MojoFailureException {
|
||||
if (System.getProperties().containsKey(PROPERTY_NAME_WF_SOURCE_DIR) &&
|
||||
!System.getProperties().containsKey(PROPERTY_NAME_SANDBOX_NAME)) {
|
||||
String generatedSandboxName = generateSandboxName(System.getProperties().getProperty(
|
||||
PROPERTY_NAME_WF_SOURCE_DIR));
|
||||
if (generatedSandboxName!=null) {
|
||||
System.getProperties().setProperty(PROPERTY_NAME_SANDBOX_NAME,
|
||||
generatedSandboxName);
|
||||
} else {
|
||||
System.out.println("unable to generate sandbox name from path: " +
|
||||
System.getProperties().getProperty(PROPERTY_NAME_WF_SOURCE_DIR));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates sandbox name from workflow source directory.
|
||||
* @param wfSourceDir
|
||||
* @return generated sandbox name
|
||||
*/
|
||||
private String generateSandboxName(String wfSourceDir) {
|
||||
// utilize all dir names until finding one of the limiters
|
||||
List<String> sandboxNameParts = new ArrayList<String>();
|
||||
String[] tokens = StringUtils.split(wfSourceDir, File.separatorChar);
|
||||
ArrayUtils.reverse(tokens);
|
||||
if (tokens.length>0) {
|
||||
for (String token : tokens) {
|
||||
for (String limiter : limiters) {
|
||||
if (limiter.equals(token)) {
|
||||
return sandboxNameParts.size()>0?
|
||||
StringUtils.join(sandboxNameParts.toArray()):null;
|
||||
}
|
||||
}
|
||||
if (sandboxNameParts.size()>0) {
|
||||
sandboxNameParts.add(0, File.separator);
|
||||
}
|
||||
sandboxNameParts.add(0, token);
|
||||
}
|
||||
return StringUtils.join(sandboxNameParts.toArray());
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,436 @@
|
|||
/**
|
||||
*
|
||||
* Licensed under the Educational Community License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.opensource.org/licenses/ecl2.php
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package eu.dnetlib.maven.plugin.properties;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.maven.plugin.AbstractMojo;
|
||||
import org.apache.maven.plugin.MojoExecutionException;
|
||||
import org.apache.maven.plugin.MojoFailureException;
|
||||
import org.apache.maven.project.MavenProject;
|
||||
import org.springframework.core.io.DefaultResourceLoader;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.core.io.ResourceLoader;
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
|
||||
|
||||
/**
|
||||
* Writes project properties for the keys listed in specified properties files.
|
||||
* Based on:
|
||||
* http://site.kuali.org/maven/plugins/properties-maven-plugin/1.3.2/write-project-properties-mojo.html
|
||||
|
||||
* @author mhorst
|
||||
* @goal write-project-properties
|
||||
*/
|
||||
public class WritePredefinedProjectProperties extends AbstractMojo {
|
||||
|
||||
private static final String CR = "\r";
|
||||
private static final String LF = "\n";
|
||||
private static final String TAB = "\t";
|
||||
protected static final String PROPERTY_PREFIX_ENV = "env.";
|
||||
private static final String ENCODING_UTF8 = "utf8";
|
||||
|
||||
/**
|
||||
* @parameter property="properties.includePropertyKeysFromFiles"
|
||||
*/
|
||||
private String[] includePropertyKeysFromFiles;
|
||||
|
||||
/**
|
||||
* @parameter default-value="${project}"
|
||||
* @required
|
||||
* @readonly
|
||||
*/
|
||||
protected MavenProject project;
|
||||
|
||||
/**
|
||||
* The file that properties will be written to
|
||||
*
|
||||
* @parameter property="properties.outputFile"
|
||||
* default-value="${project.build.directory}/properties/project.properties";
|
||||
* @required
|
||||
*/
|
||||
protected File outputFile;
|
||||
|
||||
/**
|
||||
* If true, the plugin will silently ignore any non-existent properties files, and the build will continue
|
||||
*
|
||||
* @parameter property="properties.quiet" default-value="true"
|
||||
*/
|
||||
private boolean quiet;
|
||||
|
||||
/**
|
||||
* Comma separated list of characters to escape when writing property values. cr=carriage return, lf=linefeed,
|
||||
* tab=tab. Any other values are taken literally.
|
||||
*
|
||||
* @parameter default-value="cr,lf,tab" property="properties.escapeChars"
|
||||
*/
|
||||
private String escapeChars;
|
||||
|
||||
/**
|
||||
* If true, the plugin will include system properties when writing the properties file. System properties override
|
||||
* both environment variables and project properties.
|
||||
*
|
||||
* @parameter default-value="false" property="properties.includeSystemProperties"
|
||||
*/
|
||||
private boolean includeSystemProperties;
|
||||
|
||||
/**
|
||||
* If true, the plugin will include environment variables when writing the properties file. Environment variables
|
||||
* are prefixed with "env". Environment variables override project properties.
|
||||
*
|
||||
* @parameter default-value="false" property="properties.includeEnvironmentVariables"
|
||||
*/
|
||||
private boolean includeEnvironmentVariables;
|
||||
|
||||
/**
|
||||
* Comma separated set of properties to exclude when writing the properties file
|
||||
*
|
||||
* @parameter property="properties.exclude"
|
||||
*/
|
||||
private String exclude;
|
||||
|
||||
/**
|
||||
* Comma separated set of properties to write to the properties file. If provided, only the properties matching
|
||||
* those supplied here will be written to the properties file.
|
||||
*
|
||||
* @parameter property="properties.include"
|
||||
*/
|
||||
private String include;
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.apache.maven.plugin.AbstractMojo#execute()
|
||||
*/
|
||||
@Override
|
||||
@SuppressFBWarnings({"NP_UNWRITTEN_FIELD","UWF_UNWRITTEN_FIELD"})
|
||||
public void execute() throws MojoExecutionException, MojoFailureException {
|
||||
Properties properties = new Properties();
|
||||
// Add project properties
|
||||
properties.putAll(project.getProperties());
|
||||
if (includeEnvironmentVariables) {
|
||||
// Add environment variables, overriding any existing properties with the same key
|
||||
properties.putAll(getEnvironmentVariables());
|
||||
}
|
||||
if (includeSystemProperties) {
|
||||
// Add system properties, overriding any existing properties with the same key
|
||||
properties.putAll(System.getProperties());
|
||||
}
|
||||
|
||||
// Remove properties as appropriate
|
||||
trim(properties, exclude, include);
|
||||
|
||||
String comment = "# " + new Date() + "\n";
|
||||
List<String> escapeTokens = getEscapeChars(escapeChars);
|
||||
|
||||
getLog().info("Creating " + outputFile);
|
||||
writeProperties(outputFile, comment, properties, escapeTokens);
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides environment variables.
|
||||
* @return environment variables
|
||||
*/
|
||||
protected static Properties getEnvironmentVariables() {
|
||||
Properties props = new Properties();
|
||||
for (Entry<String, String> entry : System.getenv().entrySet()) {
|
||||
props.setProperty(PROPERTY_PREFIX_ENV + entry.getKey(), entry.getValue());
|
||||
}
|
||||
return props;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes properties which should not be written.
|
||||
* @param properties
|
||||
* @param omitCSV
|
||||
* @param includeCSV
|
||||
* @throws MojoExecutionException
|
||||
*/
|
||||
protected void trim(Properties properties, String omitCSV, String includeCSV) throws MojoExecutionException {
|
||||
List<String> omitKeys = getListFromCSV(omitCSV);
|
||||
for (String key : omitKeys) {
|
||||
properties.remove(key);
|
||||
}
|
||||
|
||||
List<String> includeKeys = getListFromCSV(includeCSV);
|
||||
// mh: including keys from predefined properties
|
||||
if (includePropertyKeysFromFiles!=null && includePropertyKeysFromFiles.length>0) {
|
||||
for (String currentIncludeLoc : includePropertyKeysFromFiles) {
|
||||
if (validate(currentIncludeLoc)) {
|
||||
Properties p = getProperties(currentIncludeLoc);
|
||||
for (String key : p.stringPropertyNames()) {
|
||||
includeKeys.add(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (includeKeys!=null && !includeKeys.isEmpty()) {
|
||||
// removing only when include keys provided
|
||||
Set<String> keys = properties.stringPropertyNames();
|
||||
for (String key : keys) {
|
||||
if (!includeKeys.contains(key)) {
|
||||
properties.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether file exists.
|
||||
* @param location
|
||||
* @return true when exists, false otherwise.
|
||||
*/
|
||||
protected boolean exists(String location) {
|
||||
if (StringUtils.isBlank(location)) {
|
||||
return false;
|
||||
}
|
||||
File file = new File(location);
|
||||
if (file.exists()) {
|
||||
return true;
|
||||
}
|
||||
ResourceLoader loader = new DefaultResourceLoader();
|
||||
Resource resource = loader.getResource(location);
|
||||
return resource.exists();
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates resource location.
|
||||
* @param location
|
||||
* @return true when valid, false otherwise
|
||||
* @throws MojoExecutionException
|
||||
*/
|
||||
protected boolean validate(String location) throws MojoExecutionException {
|
||||
boolean exists = exists(location);
|
||||
if (exists) {
|
||||
return true;
|
||||
}
|
||||
if (quiet) {
|
||||
getLog().info("Ignoring non-existent properties file '" + location + "'");
|
||||
return false;
|
||||
} else {
|
||||
throw new MojoExecutionException("Non-existent properties file '" + location + "'");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides input stream.
|
||||
* @param location
|
||||
* @return input stream
|
||||
* @throws IOException
|
||||
*/
|
||||
protected InputStream getInputStream(String location) throws IOException {
|
||||
File file = new File(location);
|
||||
if (file.exists()) {
|
||||
return new FileInputStream(location);
|
||||
}
|
||||
ResourceLoader loader = new DefaultResourceLoader();
|
||||
Resource resource = loader.getResource(location);
|
||||
return resource.getInputStream();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates properties for given location.
|
||||
* @param location
|
||||
* @return properties for given location
|
||||
* @throws MojoExecutionException
|
||||
*/
|
||||
protected Properties getProperties(String location) throws MojoExecutionException {
|
||||
InputStream in = null;
|
||||
try {
|
||||
Properties properties = new Properties();
|
||||
in = getInputStream(location);
|
||||
if (location.toLowerCase().endsWith(".xml")) {
|
||||
properties.loadFromXML(in);
|
||||
} else {
|
||||
properties.load(in);
|
||||
}
|
||||
return properties;
|
||||
} catch (IOException e) {
|
||||
throw new MojoExecutionException("Error reading properties file " + location, e);
|
||||
} finally {
|
||||
IOUtils.closeQuietly(in);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides escape characters.
|
||||
* @param escapeChars
|
||||
* @return escape characters
|
||||
*/
|
||||
protected List<String> getEscapeChars(String escapeChars) {
|
||||
List<String> tokens = getListFromCSV(escapeChars);
|
||||
List<String> realTokens = new ArrayList<String>();
|
||||
for (String token : tokens) {
|
||||
String realToken = getRealToken(token);
|
||||
realTokens.add(realToken);
|
||||
}
|
||||
return realTokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides real token.
|
||||
* @param token
|
||||
* @return real token
|
||||
*/
|
||||
protected String getRealToken(String token) {
|
||||
if (token.equalsIgnoreCase("CR")) {
|
||||
return CR;
|
||||
} else if (token.equalsIgnoreCase("LF")) {
|
||||
return LF;
|
||||
} else if (token.equalsIgnoreCase("TAB")) {
|
||||
return TAB;
|
||||
} else {
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns content.
|
||||
* @param comment
|
||||
* @param properties
|
||||
* @param escapeTokens
|
||||
* @return content
|
||||
*/
|
||||
protected String getContent(String comment, Properties properties, List<String> escapeTokens) {
|
||||
List<String> names = new ArrayList<String>(properties.stringPropertyNames());
|
||||
Collections.sort(names);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
if (!StringUtils.isBlank(comment)) {
|
||||
sb.append(comment);
|
||||
}
|
||||
for (String name : names) {
|
||||
String value = properties.getProperty(name);
|
||||
String escapedValue = escape(value, escapeTokens);
|
||||
sb.append(name + "=" + escapedValue + "\n");
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes properties to given file.
|
||||
* @param file
|
||||
* @param comment
|
||||
* @param properties
|
||||
* @param escapeTokens
|
||||
* @throws MojoExecutionException
|
||||
*/
|
||||
protected void writeProperties(File file, String comment, Properties properties, List<String> escapeTokens)
|
||||
throws MojoExecutionException {
|
||||
try {
|
||||
String content = getContent(comment, properties, escapeTokens);
|
||||
FileUtils.writeStringToFile(file, content, ENCODING_UTF8);
|
||||
} catch (IOException e) {
|
||||
throw new MojoExecutionException("Error creating properties file", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Escapes characters.
|
||||
* @param s
|
||||
* @param escapeChars
|
||||
* @return
|
||||
*/
|
||||
protected String escape(String s, List<String> escapeChars) {
|
||||
String result = s;
|
||||
for (String escapeChar : escapeChars) {
|
||||
result = result.replace(escapeChar, getReplacementToken(escapeChar));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides replacement token.
|
||||
* @param escapeChar
|
||||
* @return replacement token
|
||||
*/
|
||||
protected String getReplacementToken(String escapeChar) {
|
||||
if (escapeChar.equals(CR)) {
|
||||
return "\\r";
|
||||
} else if (escapeChar.equals(LF)) {
|
||||
return "\\n";
|
||||
} else if (escapeChar.equals(TAB)) {
|
||||
return "\\t";
|
||||
} else {
|
||||
return "\\" + escapeChar;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns list from csv.
|
||||
* @param csv
|
||||
* @return list of values generated from CSV
|
||||
*/
|
||||
protected static final List<String> getListFromCSV(String csv) {
|
||||
if (StringUtils.isBlank(csv)) {
|
||||
return new ArrayList<String>();
|
||||
}
|
||||
List<String> list = new ArrayList<String>();
|
||||
String[] tokens = StringUtils.split(csv, ",");
|
||||
for (String token : tokens) {
|
||||
list.add(token.trim());
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public void setIncludeSystemProperties(boolean includeSystemProperties) {
|
||||
this.includeSystemProperties = includeSystemProperties;
|
||||
}
|
||||
|
||||
public void setEscapeChars(String escapeChars) {
|
||||
this.escapeChars = escapeChars;
|
||||
}
|
||||
|
||||
public void setIncludeEnvironmentVariables(boolean includeEnvironmentVariables) {
|
||||
this.includeEnvironmentVariables = includeEnvironmentVariables;
|
||||
}
|
||||
|
||||
public void setExclude(String exclude) {
|
||||
this.exclude = exclude;
|
||||
}
|
||||
|
||||
public void setInclude(String include) {
|
||||
this.include = include;
|
||||
}
|
||||
|
||||
public void setQuiet(boolean quiet) {
|
||||
this.quiet = quiet;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets property files for which keys properties should be included.
|
||||
* @param includePropertyKeysFromFiles
|
||||
*/
|
||||
public void setIncludePropertyKeysFromFiles(
|
||||
String[] includePropertyKeysFromFiles) {
|
||||
if (includePropertyKeysFromFiles!=null) {
|
||||
this.includePropertyKeysFromFiles = Arrays.copyOf(
|
||||
includePropertyKeysFromFiles,
|
||||
includePropertyKeysFromFiles.length);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
package eu.dnetlib.maven.plugin.properties;
|
||||
|
||||
import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_SANDBOX_NAME;
|
||||
import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_WF_SOURCE_DIR;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* @author mhorst
|
||||
*
|
||||
*/
|
||||
public class GenerateOoziePropertiesMojoTest {
|
||||
|
||||
private GenerateOoziePropertiesMojo mojo = new GenerateOoziePropertiesMojo();
|
||||
|
||||
@Before
|
||||
public void clearSystemProperties() {
|
||||
System.clearProperty(PROPERTY_NAME_SANDBOX_NAME);
|
||||
System.clearProperty(PROPERTY_NAME_WF_SOURCE_DIR);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteEmpty() throws Exception {
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteSandboxNameAlreadySet() throws Exception {
|
||||
// given
|
||||
String workflowSourceDir = "eu/dnetlib/iis/wf/transformers";
|
||||
String sandboxName = "originalSandboxName";
|
||||
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||
System.setProperty(PROPERTY_NAME_SANDBOX_NAME, sandboxName);
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertEquals(sandboxName, System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteEmptyWorkflowSourceDir() throws Exception {
|
||||
// given
|
||||
String workflowSourceDir = "";
|
||||
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteNullSandboxNameGenerated() throws Exception {
|
||||
// given
|
||||
String workflowSourceDir = "eu/dnetlib/iis/";
|
||||
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecute() throws Exception {
|
||||
// given
|
||||
String workflowSourceDir = "eu/dnetlib/iis/wf/transformers";
|
||||
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertEquals("wf/transformers", System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteWithoutRoot() throws Exception {
|
||||
// given
|
||||
String workflowSourceDir = "wf/transformers";
|
||||
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertEquals("wf/transformers", System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,365 @@
|
|||
package eu.dnetlib.maven.plugin.properties;
|
||||
|
||||
import static eu.dnetlib.maven.plugin.properties.WritePredefinedProjectProperties.PROPERTY_PREFIX_ENV;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.maven.plugin.MojoExecutionException;
|
||||
import org.apache.maven.project.MavenProject;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
|
||||
/**
|
||||
* @author mhorst
|
||||
*
|
||||
*/
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class WritePredefinedProjectPropertiesTest {
|
||||
|
||||
@Rule
|
||||
public TemporaryFolder testFolder = new TemporaryFolder();
|
||||
|
||||
@Mock
|
||||
private MavenProject mavenProject;
|
||||
|
||||
private WritePredefinedProjectProperties mojo;
|
||||
|
||||
@Before
|
||||
public void init() {
|
||||
mojo = new WritePredefinedProjectProperties();
|
||||
mojo.outputFile = getPropertiesFileLocation();
|
||||
mojo.project = mavenProject;
|
||||
doReturn(new Properties()).when(mavenProject).getProperties();
|
||||
}
|
||||
|
||||
// ----------------------------------- TESTS ---------------------------------------------
|
||||
|
||||
@Test
|
||||
public void testExecuteEmpty() throws Exception {
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertEquals(0, storedProperties.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteWithProjectProperties() throws Exception {
|
||||
// given
|
||||
String key = "projectPropertyKey";
|
||||
String value = "projectPropertyValue";
|
||||
Properties projectProperties = new Properties();
|
||||
projectProperties.setProperty(key, value);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertEquals(1, storedProperties.size());
|
||||
assertTrue(storedProperties.containsKey(key));
|
||||
assertEquals(value, storedProperties.getProperty(key));
|
||||
}
|
||||
|
||||
@Test(expected=MojoExecutionException.class)
|
||||
public void testExecuteWithProjectPropertiesAndInvalidOutputFile() throws Exception {
|
||||
// given
|
||||
String key = "projectPropertyKey";
|
||||
String value = "projectPropertyValue";
|
||||
Properties projectProperties = new Properties();
|
||||
projectProperties.setProperty(key, value);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
mojo.outputFile = testFolder.getRoot();
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteWithProjectPropertiesExclusion() throws Exception {
|
||||
// given
|
||||
String key = "projectPropertyKey";
|
||||
String value = "projectPropertyValue";
|
||||
String excludedKey = "excludedPropertyKey";
|
||||
String excludedValue = "excludedPropertyValue";
|
||||
Properties projectProperties = new Properties();
|
||||
projectProperties.setProperty(key, value);
|
||||
projectProperties.setProperty(excludedKey, excludedValue);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
mojo.setExclude(excludedKey);
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertEquals(1, storedProperties.size());
|
||||
assertTrue(storedProperties.containsKey(key));
|
||||
assertEquals(value, storedProperties.getProperty(key));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteWithProjectPropertiesInclusion() throws Exception {
|
||||
// given
|
||||
String key = "projectPropertyKey";
|
||||
String value = "projectPropertyValue";
|
||||
String includedKey = "includedPropertyKey";
|
||||
String includedValue = "includedPropertyValue";
|
||||
Properties projectProperties = new Properties();
|
||||
projectProperties.setProperty(key, value);
|
||||
projectProperties.setProperty(includedKey, includedValue);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
mojo.setInclude(includedKey);
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertEquals(1, storedProperties.size());
|
||||
assertTrue(storedProperties.containsKey(includedKey));
|
||||
assertEquals(includedValue, storedProperties.getProperty(includedKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteIncludingPropertyKeysFromFile() throws Exception {
|
||||
// given
|
||||
String key = "projectPropertyKey";
|
||||
String value = "projectPropertyValue";
|
||||
String includedKey = "includedPropertyKey";
|
||||
String includedValue = "includedPropertyValue";
|
||||
Properties projectProperties = new Properties();
|
||||
projectProperties.setProperty(key, value);
|
||||
projectProperties.setProperty(includedKey, includedValue);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
|
||||
File includedPropertiesFile = new File(testFolder.getRoot(), "included.properties");
|
||||
Properties includedProperties = new Properties();
|
||||
includedProperties.setProperty(includedKey, "irrelevantValue");
|
||||
includedProperties.store(new FileWriter(includedPropertiesFile), null);
|
||||
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertEquals(1, storedProperties.size());
|
||||
assertTrue(storedProperties.containsKey(includedKey));
|
||||
assertEquals(includedValue, storedProperties.getProperty(includedKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteIncludingPropertyKeysFromClasspathResource() throws Exception {
|
||||
// given
|
||||
String key = "projectPropertyKey";
|
||||
String value = "projectPropertyValue";
|
||||
String includedKey = "includedPropertyKey";
|
||||
String includedValue = "includedPropertyValue";
|
||||
Properties projectProperties = new Properties();
|
||||
projectProperties.setProperty(key, value);
|
||||
projectProperties.setProperty(includedKey, includedValue);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {"/eu/dnetlib/maven/plugin/properties/included.properties"});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertEquals(1, storedProperties.size());
|
||||
assertTrue(storedProperties.containsKey(includedKey));
|
||||
assertEquals(includedValue, storedProperties.getProperty(includedKey));
|
||||
}
|
||||
|
||||
@Test(expected=MojoExecutionException.class)
|
||||
public void testExecuteIncludingPropertyKeysFromBlankLocation() throws Exception {
|
||||
// given
|
||||
String key = "projectPropertyKey";
|
||||
String value = "projectPropertyValue";
|
||||
String includedKey = "includedPropertyKey";
|
||||
String includedValue = "includedPropertyValue";
|
||||
Properties projectProperties = new Properties();
|
||||
projectProperties.setProperty(key, value);
|
||||
projectProperties.setProperty(includedKey, includedValue);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {""});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteIncludingPropertyKeysFromXmlFile() throws Exception {
|
||||
// given
|
||||
String key = "projectPropertyKey";
|
||||
String value = "projectPropertyValue";
|
||||
String includedKey = "includedPropertyKey";
|
||||
String includedValue = "includedPropertyValue";
|
||||
Properties projectProperties = new Properties();
|
||||
projectProperties.setProperty(key, value);
|
||||
projectProperties.setProperty(includedKey, includedValue);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
|
||||
File includedPropertiesFile = new File(testFolder.getRoot(), "included.xml");
|
||||
Properties includedProperties = new Properties();
|
||||
includedProperties.setProperty(includedKey, "irrelevantValue");
|
||||
includedProperties.storeToXML(new FileOutputStream(includedPropertiesFile), null);
|
||||
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertEquals(1, storedProperties.size());
|
||||
assertTrue(storedProperties.containsKey(includedKey));
|
||||
assertEquals(includedValue, storedProperties.getProperty(includedKey));
|
||||
}
|
||||
|
||||
@Test(expected=MojoExecutionException.class)
|
||||
public void testExecuteIncludingPropertyKeysFromInvalidXmlFile() throws Exception {
|
||||
// given
|
||||
String key = "projectPropertyKey";
|
||||
String value = "projectPropertyValue";
|
||||
String includedKey = "includedPropertyKey";
|
||||
String includedValue = "includedPropertyValue";
|
||||
Properties projectProperties = new Properties();
|
||||
projectProperties.setProperty(key, value);
|
||||
projectProperties.setProperty(includedKey, includedValue);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
|
||||
File includedPropertiesFile = new File(testFolder.getRoot(), "included.xml");
|
||||
Properties includedProperties = new Properties();
|
||||
includedProperties.setProperty(includedKey, "irrelevantValue");
|
||||
includedProperties.store(new FileOutputStream(includedPropertiesFile), null);
|
||||
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteWithQuietModeOn() throws Exception {
|
||||
// given
|
||||
mojo.setQuiet(true);
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertEquals(0, storedProperties.size());
|
||||
}
|
||||
|
||||
@Test(expected=MojoExecutionException.class)
|
||||
public void testExecuteIncludingPropertyKeysFromInvalidFile() throws Exception {
|
||||
// given
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteWithEnvironmentProperties() throws Exception {
|
||||
// given
|
||||
mojo.setIncludeEnvironmentVariables(true);
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertTrue(storedProperties.size() > 0);
|
||||
for (Object currentKey : storedProperties.keySet()) {
|
||||
assertTrue(((String)currentKey).startsWith(PROPERTY_PREFIX_ENV));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteWithSystemProperties() throws Exception {
|
||||
// given
|
||||
String key = "systemPropertyKey";
|
||||
String value = "systemPropertyValue";
|
||||
System.setProperty(key, value);
|
||||
mojo.setIncludeSystemProperties(true);
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertTrue(storedProperties.size() > 0);
|
||||
assertTrue(storedProperties.containsKey(key));
|
||||
assertEquals(value, storedProperties.getProperty(key));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteWithSystemPropertiesAndEscapeChars() throws Exception {
|
||||
// given
|
||||
String key = "systemPropertyKey ";
|
||||
String value = "systemPropertyValue";
|
||||
System.setProperty(key, value);
|
||||
mojo.setIncludeSystemProperties(true);
|
||||
String escapeChars = "cr,lf,tab,|";
|
||||
mojo.setEscapeChars(escapeChars);
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
||||
// assert
|
||||
assertTrue(mojo.outputFile.exists());
|
||||
Properties storedProperties = getStoredProperties();
|
||||
assertTrue(storedProperties.size() > 0);
|
||||
assertFalse(storedProperties.containsKey(key));
|
||||
assertTrue(storedProperties.containsKey(key.trim()));
|
||||
assertEquals(value, storedProperties.getProperty(key.trim()));
|
||||
}
|
||||
|
||||
// ----------------------------------- PRIVATE -------------------------------------------
|
||||
|
||||
private File getPropertiesFileLocation() {
|
||||
return new File(testFolder.getRoot(), "test.properties");
|
||||
}
|
||||
|
||||
private Properties getStoredProperties() throws FileNotFoundException, IOException {
|
||||
Properties properties = new Properties();
|
||||
properties.load(new FileInputStream(getPropertiesFileLocation()));
|
||||
return properties;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
includedPropertyKey=irrelevantValue
|
||||
|
|
@ -9,7 +9,8 @@
|
|||
<artifactId>dhp-build</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
<modules>
|
||||
|
||||
<module>dhp-build-assembly-resources</module>
|
||||
<module>dhp-build-properties-maven-plugin</module>
|
||||
</modules>
|
||||
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>dhp-workflows</artifactId>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>dhp-distcp</artifactId>
|
||||
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
<configuration>
|
||||
<property>
|
||||
<name>jobTracker</name>
|
||||
<value>yarnRM</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>nameNode</name>
|
||||
<value>hdfs://nameservice1</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>sourceNN</name>
|
||||
<value>webhdfs://namenode2.hadoop.dm.openaire.eu:50071</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.use.system.libpath</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
<workflow-app name="distcp" xmlns="uri:oozie:workflow:0.5">
|
||||
<parameters>
|
||||
<property>
|
||||
<name>sourceNN</name>
|
||||
<description>the source name node</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>sourcePath</name>
|
||||
<description>the source path</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>targetPath</name>
|
||||
<description>the target path</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>hbase_dump_distcp_memory_mb</name>
|
||||
<value>6144</value>
|
||||
<description>memory for distcp action copying InfoSpace dump from remote cluster</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>hbase_dump_distcp_num_maps</name>
|
||||
<value>1</value>
|
||||
<description>maximum number of simultaneous copies of InfoSpace dump from remote location</description>
|
||||
</property>
|
||||
</parameters>
|
||||
|
||||
<start to="distcp"/>
|
||||
|
||||
<kill name="Kill">
|
||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||
</kill>
|
||||
|
||||
<action name="distcp">
|
||||
<distcp xmlns="uri:oozie:distcp-action:0.2">
|
||||
<arg>-Dmapreduce.map.memory.mb=${hbase_dump_distcp_memory_mb}</arg>
|
||||
<arg>-pb</arg>
|
||||
<arg>-m ${hbase_dump_distcp_num_maps}</arg>
|
||||
<arg>${sourceNN}/${sourcePath}</arg>
|
||||
<arg>${nameNode}/${targetPath}</arg>
|
||||
</distcp>
|
||||
<ok to="End" />
|
||||
<error to="Kill" />
|
||||
</action>
|
||||
|
||||
<end name="End"/>
|
||||
</workflow-app>
|
||||
|
|
@ -1,16 +1,604 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<relativePath>../</relativePath>
|
||||
</parent>
|
||||
|
||||
<artifactId>dhp-workflows</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
|
||||
<modules>
|
||||
<module>dhp-aggregation</module>
|
||||
<module>dhp-distcp</module>
|
||||
</modules>
|
||||
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>iis-releases</id>
|
||||
<name>iis releases plugin repository</name>
|
||||
<url>http://maven.ceon.pl/artifactory/iis-releases</url>
|
||||
<layout>default</layout>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
<properties>
|
||||
<maven.build.timestamp.format>yyyy-MM-dd_HH_mm</maven.build.timestamp.format>
|
||||
<!-- default Oozie installer properties requred to be defined at pom.xml level -->
|
||||
<!-- other project properties are defined in project-default.properties -->
|
||||
<oozie.package.file.name>oozie-package</oozie.package.file.name>
|
||||
<!-- notice: sandboxName is generated based on workflow.source.dir property -->
|
||||
<workflow.source.dir>src/test/resources/define/path/pointing/to/directory/holding/oozie_app</workflow.source.dir>
|
||||
<oozieAppDir>oozie_app</oozieAppDir>
|
||||
<queueName>default</queueName>
|
||||
<importerQueueName>default</importerQueueName>
|
||||
<oozieLauncherQueueName>default</oozieLauncherQueueName>
|
||||
<primed.dir>primed</primed.dir>
|
||||
|
||||
<oozie.package.dependencies.include.scope>runtime</oozie.package.dependencies.include.scope>
|
||||
<oozie.package.dependencies.exclude.scope></oozie.package.dependencies.exclude.scope>
|
||||
<oozie.package.skip.test.jar>true</oozie.package.skip.test.jar>
|
||||
|
||||
<dhpConnectionProperties>${user.home}/.dhp/application.properties</dhpConnectionProperties>
|
||||
|
||||
<output.dir.name>${maven.build.timestamp}</output.dir.name>
|
||||
|
||||
<projectVersion>${project.version}</projectVersion>
|
||||
<oozie.use.system.libpath>true</oozie.use.system.libpath>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-build-assembly-resources</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.oozie</groupId>
|
||||
<artifactId>oozie-client</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.schmizz</groupId>
|
||||
<artifactId>sshj</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<profiles>
|
||||
<profile>
|
||||
<!-- This profile sets properties that are required for test oozie workflows To be used only with 'oozie-package' profile -->
|
||||
<id>attach-test-resources</id>
|
||||
<properties>
|
||||
<!--overriding default scope (set to 'runtime') with the 'test' value. Test resources attached to oozie package requires all test dependencies. -->
|
||||
<oozie.package.dependencies.include.scope></oozie.package.dependencies.include.scope>
|
||||
<oozie.package.dependencies.exclude.scope>provided</oozie.package.dependencies.exclude.scope>
|
||||
<!-- Do not skip creation of test jar for priming (in oozie-package profile) -->
|
||||
<oozie.package.skip.test.jar>false</oozie.package.skip.test.jar>
|
||||
</properties>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>oozie-package</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-enforcer-plugin</artifactId>
|
||||
<version>1.4.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>enforce-connection-properties-file-existence</id>
|
||||
<phase>initialize</phase>
|
||||
<goals>
|
||||
<goal>enforce</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<requireFilesExist>
|
||||
<files>
|
||||
<file>${dhpConnectionProperties}</file>
|
||||
</files>
|
||||
<message>
|
||||
The file with connection properties could not be found. Please, create the ${dhpConnectionProperties} file or set the location to another already created file by using
|
||||
-DdhpConnectionProperties property.
|
||||
</message>
|
||||
</requireFilesExist>
|
||||
</rules>
|
||||
<fail>true</fail>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy dependencies</id>
|
||||
<phase>prepare-package</phase>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<includeScope>${oozie.package.dependencies.include.scope}</includeScope>
|
||||
<excludeScope>${oozie.package.dependencies.exclude.scope}</excludeScope>
|
||||
<silent>true</silent>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<!-- Plugin originally defined in attach-test-resources It was moved here to ensure that it will execute before priming -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-test-resources-package</id>
|
||||
<phase>prepare-package</phase>
|
||||
<goals>
|
||||
<goal>test-jar</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<skip>${oozie.package.skip.test.jar}</skip>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>eu.dnetlib.primer</groupId>
|
||||
<artifactId>primer-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>priming</id>
|
||||
<phase>prepare-package</phase>
|
||||
<goals>
|
||||
<goal>prime</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<classProviderFiles>
|
||||
<classProviderFile>${project.build.directory}/dependency/*.jar</classProviderFile>
|
||||
<classProviderFile>${project.build.directory}/*-tests.jar</classProviderFile>
|
||||
<classProviderFile>${project.build.directory}/classes</classProviderFile>
|
||||
</classProviderFiles>
|
||||
<coansysPackageDir>${project.build.directory}/dependency</coansysPackageDir>
|
||||
<destination>${project.build.directory}/${primed.dir}</destination>
|
||||
<classpath>${workflow.source.dir}</classpath>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<!-- reading job.properties to use them in .sh scripts -->
|
||||
<plugin>
|
||||
<groupId>org.kuali.maven.plugins</groupId>
|
||||
<artifactId>properties-maven-plugin</artifactId>
|
||||
<version>1.3.2</version>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-build-assembly-resources</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<!-- contains project-default.properties -->
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>reading-dhp-properties</id>
|
||||
<phase>initialize</phase>
|
||||
<goals>
|
||||
<goal>read-project-properties</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<locations>
|
||||
<location>${dhpConnectionProperties}</location>
|
||||
</locations>
|
||||
<quiet>false</quiet>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>read-default-properties</id>
|
||||
<phase>prepare-package</phase>
|
||||
<goals>
|
||||
<goal>read-project-properties</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<locations>
|
||||
<location>classpath:project-default.properties</location>
|
||||
</locations>
|
||||
<quiet>true</quiet>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>read-job-properties</id>
|
||||
<phase>prepare-package</phase>
|
||||
<goals>
|
||||
<goal>read-project-properties</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<locations>
|
||||
<param>${project.build.directory}/${primed.dir}/job.properties</param>
|
||||
<param>job-override.properties</param>
|
||||
</locations>
|
||||
<quiet>true</quiet>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-build-properties-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>generate-properties</goal>
|
||||
<!-- generates sandboxName based on workflow.source.dir when not specified as commandline parameter -->
|
||||
</goals>
|
||||
<configuration>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>write-job-properties</id>
|
||||
<phase>prepare-package</phase>
|
||||
<goals>
|
||||
<goal>write-project-properties</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputFile>target/${oozie.package.file.name}/job.properties</outputFile>
|
||||
<!-- notice: dots are not allowed for job.properties! -->
|
||||
<include>nameNode,jobTracker,queueName,importerQueueName,oozieLauncherQueueName,
|
||||
workingDir,oozieTopWfApplicationPath,oozieServiceLoc,
|
||||
sparkDriverMemory,sparkExecutorMemory,sparkExecutorCores,
|
||||
oozie.wf.application.path,projectVersion,oozie.use.system.libpath</include>
|
||||
<includeSystemProperties>true</includeSystemProperties>
|
||||
<includePropertyKeysFromFiles>
|
||||
<!-- <param>${workflow.source.dir}/job.properties</param> -->
|
||||
<param>${project.build.directory}/${primed.dir}/job.properties</param>
|
||||
<param>job-override.properties</param>
|
||||
|
||||
</includePropertyKeysFromFiles>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>pl.project13.maven</groupId>
|
||||
<artifactId>git-commit-id-plugin</artifactId>
|
||||
<version>2.1.11</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>revision</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<verbose>true</verbose>
|
||||
<dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
|
||||
<generateGitPropertiesFile>true</generateGitPropertiesFile>
|
||||
<generateGitPropertiesFilename>target/${oozie.package.file.name}/${oozieAppDir}/version.properties</generateGitPropertiesFilename>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>3.0.0</version>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-build-assembly-resources</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>assembly-oozie-installer</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<appendAssemblyId>false</appendAssemblyId>
|
||||
<finalName>${oozie.package.file.name}_shell_scripts</finalName>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>oozie-installer</descriptorRef>
|
||||
</descriptorRefs>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<!-- this plugin prepares oozie installer package-->
|
||||
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<executions>
|
||||
<!-- extracting shared resources phase -->
|
||||
<execution>
|
||||
<id>installer-copy-custom</id>
|
||||
<phase>process-resources</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<tasks>
|
||||
<property name="assembly-resources.loc" value="${maven.dependency.eu.dnetlib.dhp.dhp-build-assembly-resources.jar.path}" />
|
||||
<unjar src="${assembly-resources.loc}" dest="${project.build.directory}/assembly-resources" />
|
||||
</tasks>
|
||||
</configuration>
|
||||
</execution>
|
||||
<!-- packaging phase -->
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<configuration>
|
||||
<tasks>
|
||||
<!-- copying workflow resources -->
|
||||
<mkdir dir="target/${oozie.package.file.name}" />
|
||||
<mkdir dir="target/${oozie.package.file.name}/${oozieAppDir}" />
|
||||
<copy todir="target/${oozie.package.file.name}/${oozieAppDir}">
|
||||
<!-- <fileset dir="${workflow.source.dir}/${oozieAppDir}" /> replacing with primed dir location -->
|
||||
<fileset dir="target/${primed.dir}/${oozieAppDir}" />
|
||||
</copy>
|
||||
<!-- copying all jars to oozie lib directory -->
|
||||
<mkdir dir="target/${oozie.package.file.name}/${oozieAppDir}/lib" />
|
||||
<copy todir="target/${oozie.package.file.name}/${oozieAppDir}/lib">
|
||||
<fileset dir="${project.build.directory}/dependency" />
|
||||
</copy>
|
||||
<!-- copying current module lib -->
|
||||
<copy todir="target/${oozie.package.file.name}/${oozieAppDir}/lib">
|
||||
<fileset dir="${project.build.directory}">
|
||||
<include name="*.jar" />
|
||||
</fileset>
|
||||
</copy>
|
||||
|
||||
|
||||
<fixcrlf srcdir="target/${oozie.package.file.name}/${oozieAppDir}/" encoding="UTF-8" outputencoding="UTF-8" includes="**/*.sh,**/*.json,**/*.py,**/*.sql" eol="lf"/>
|
||||
|
||||
|
||||
<!-- creating tar.gz package -->
|
||||
<tar destfile="target/${oozie.package.file.name}.tar.gz" compression="gzip">
|
||||
<tarfileset dir="target/${oozie.package.file.name}" />
|
||||
<tarfileset dir="target/${oozie.package.file.name}_shell_scripts" filemode="0755">
|
||||
<include name="**/*.sh" />
|
||||
</tarfileset>
|
||||
<tarfileset dir="target/${oozie.package.file.name}_shell_scripts" filemode="0644">
|
||||
<exclude name="**/*.sh" />
|
||||
</tarfileset>
|
||||
</tar>
|
||||
<!-- cleanup -->
|
||||
<delete dir="target/${oozie.package.file.name}" />
|
||||
<delete dir="target/${oozie.package.file.name}_shell_scripts" />
|
||||
</tasks>
|
||||
</configuration>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>deploy</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>exec-maven-plugin</artifactId>
|
||||
<version>1.5.0</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-target-dir</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>exec</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<executable>ssh</executable>
|
||||
<arguments>
|
||||
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
|
||||
<argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
|
||||
<argument>-o StrictHostKeyChecking=no</argument>
|
||||
<argument>rm -rf ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; mkdir -p ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/</argument>
|
||||
</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>upload-oozie-package</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>exec</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<executable>scp</executable>
|
||||
<arguments>
|
||||
<argument>-P ${dhp.hadoop.frontend.port.ssh}</argument>
|
||||
<argument>-o StrictHostKeyChecking=no</argument>
|
||||
<argument>target/${oozie.package.file.name}.tar.gz</argument>
|
||||
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}:${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/${oozie.package.file.name}.tar.gz</argument>
|
||||
</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>extract-and-upload-to-hdfs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>exec</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<executable>ssh</executable>
|
||||
<!-- <outputFile>target/redirected_upload.log</outputFile> -->
|
||||
<arguments>
|
||||
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
|
||||
<argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
|
||||
<argument>-o StrictHostKeyChecking=no</argument>
|
||||
<argument>cd ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; </argument>
|
||||
<argument>tar -zxf oozie-package.tar.gz; </argument>
|
||||
<argument>rm ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/oozie-package.tar.gz; </argument>
|
||||
<argument>./upload_workflow.sh</argument>
|
||||
</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>run</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>exec-maven-plugin</artifactId>
|
||||
<version>1.5.0</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>run-job</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>exec</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<executable>ssh</executable>
|
||||
<!-- this file will be used by test verification profile reading job identifier -->
|
||||
<outputFile>${oozie.execution.log.file.location}</outputFile>
|
||||
<arguments>
|
||||
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
|
||||
<argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
|
||||
<argument>-o StrictHostKeyChecking=no</argument>
|
||||
<argument>cd ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; </argument>
|
||||
<argument>./run_workflow.sh</argument>
|
||||
</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>show-run-log-on-stdout</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>exec</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<executable>cat</executable>
|
||||
<arguments>
|
||||
<argument>${oozie.execution.log.file.location}</argument>
|
||||
</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
<profile>
|
||||
<!-- this profile is handling unit and integration test definitions of all child modules -->
|
||||
<id>child-tests</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>integration-test-package</id>
|
||||
<phase>integration-test</phase>
|
||||
<goals>
|
||||
<goal>test-jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludedGroups>eu.dnetlib.iis.common.IntegrationTest</excludedGroups>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-failsafe-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>default-integration-test</id>
|
||||
<configuration>
|
||||
<argLine>-Xmx1024m</argLine>
|
||||
<systemPropertiesVariables>
|
||||
<!-- if dhpConnectionProperties is not defined, then ${user.home}/.dhp/integration-test.properties will be used -->
|
||||
<dhpConnectionProperties>${dhpConnectionProperties}</dhpConnectionProperties>
|
||||
<output.dir.name>${output.dir.name}</output.dir.name>
|
||||
</systemPropertiesVariables>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<activation>
|
||||
<file>
|
||||
<missing>src/main/resources/parent.marker</missing>
|
||||
</file>
|
||||
</activation>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself. -->
|
||||
<plugin>
|
||||
<groupId>org.eclipse.m2e</groupId>
|
||||
<artifactId>lifecycle-mapping</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<configuration>
|
||||
<lifecycleMappingMetadata>
|
||||
<pluginExecutions>
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>
|
||||
org.kuali.maven.plugins
|
||||
</groupId>
|
||||
<artifactId>
|
||||
properties-maven-plugin
|
||||
</artifactId>
|
||||
<versionRange>
|
||||
[1.3.2,)
|
||||
</versionRange>
|
||||
<goals>
|
||||
<goal>
|
||||
read-project-properties
|
||||
</goal>
|
||||
<goal>
|
||||
write-project-properties
|
||||
</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore />
|
||||
</action>
|
||||
</pluginExecution>
|
||||
<!-- copy-dependency plugin -->
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<versionRange>[1.0.0,)</versionRange>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore />
|
||||
</action>
|
||||
</pluginExecution>
|
||||
</pluginExecutions>
|
||||
</lifecycleMappingMetadata>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
</project>
|
||||
|
|
|
|||
94
pom.xml
94
pom.xml
|
|
@ -20,7 +20,7 @@
|
|||
</licenses>
|
||||
|
||||
<modules>
|
||||
<!--<module>dhp-build</module>-->
|
||||
<module>dhp-build</module>
|
||||
<module>dhp-common</module>
|
||||
<module>dhp-workflows</module>
|
||||
<module>dhp-applications</module>
|
||||
|
|
@ -60,6 +60,8 @@
|
|||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
|
||||
<!--
|
||||
<repository>
|
||||
<id>dnet45-bootstrap-release</id>
|
||||
<name>dnet45 bootstrap release</name>
|
||||
|
|
@ -72,6 +74,7 @@
|
|||
<enabled>true</enabled>
|
||||
</releases>
|
||||
</repository>
|
||||
-->
|
||||
|
||||
<repository>
|
||||
<id>cloudera</id>
|
||||
|
|
@ -142,6 +145,26 @@
|
|||
<artifactId>commons-cli</artifactId>
|
||||
<version>1.4</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>net.schmizz</groupId>
|
||||
<artifactId>sshj</artifactId>
|
||||
<version>0.10.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.oozie</groupId>
|
||||
<artifactId>oozie-client</artifactId>
|
||||
<version>${dhp.oozie.version}</version>
|
||||
<exclusions>
|
||||
<!-- conflicts -->
|
||||
<exclusion>
|
||||
<artifactId>slf4j-simple</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
|
|
@ -192,6 +215,30 @@
|
|||
<redirectTestOutputToFile>true</redirectTestOutputToFile>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-failsafe-plugin</artifactId>
|
||||
<version>2.19.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>default-integration-test</id>
|
||||
<goals>
|
||||
<goal>integration-test</goal>
|
||||
<goal>verify</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<groups>eu.dnetlib.dhp.common.IntegrationTest</groups>
|
||||
<includes>
|
||||
<include>**/*Test.java</include>
|
||||
</includes>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
|
|
@ -205,7 +252,13 @@
|
|||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>3.0.0</version>
|
||||
</plugin>
|
||||
<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself. -->
|
||||
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>build-helper-maven-plugin</artifactId>
|
||||
<version>1.12</version>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
|
||||
|
|
@ -215,8 +268,42 @@
|
|||
<artifactId>maven-release-plugin</artifactId>
|
||||
<version>2.5.3</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.7.9</version>
|
||||
<configuration>
|
||||
<excludes>
|
||||
<exclude>**/schemas/*</exclude>
|
||||
<exclude>**/com/cloudera/**/*</exclude>
|
||||
<exclude>**/org/apache/avro/io/**/*</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>default-prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>default-report</id>
|
||||
<phase>prepare-package</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
<extensions>
|
||||
<extension>
|
||||
<groupId>org.apache.maven.wagon</groupId>
|
||||
<artifactId>wagon-ssh</artifactId>
|
||||
<version>2.10</version>
|
||||
</extension>
|
||||
</extensions>
|
||||
|
||||
</build>
|
||||
|
||||
|
|
@ -250,9 +337,12 @@
|
|||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
|
||||
<dhp.cdh.version>cdh5.9.2</dhp.cdh.version>
|
||||
<dhp.hadoop.version>2.6.0-${dhp.cdh.version}</dhp.hadoop.version>
|
||||
<dhp.oozie.version>4.1.0-${dhp.cdh.version}</dhp.oozie.version>
|
||||
<dhp.spark.version>2.2.0</dhp.spark.version>
|
||||
|
||||
<dhp.commons.lang.version>3.5</dhp.commons.lang.version>
|
||||
<scala.version>2.11.8</scala.version>
|
||||
</properties>
|
||||
|
|
|
|||
Loading…
Reference in New Issue