integrating the oozie workflow build/deploy/run mechanism, took inspiration from iis
This commit is contained in:
parent
bbb87d0e3d
commit
27db5afdad
|
|
@ -0,0 +1,7 @@
|
||||||
|
Module utilized by `dhp-workflows`.
|
||||||
|
|
||||||
|
Contains all required resources by this parent module:
|
||||||
|
|
||||||
|
* assembly XML definitions
|
||||||
|
* build shell scripts
|
||||||
|
* oozie package commands for uploading, running and monitoring oozie workflows
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||||
|
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<parent>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>dhp-build</artifactId>
|
||||||
|
<version>1.0.0-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<artifactId>dhp-build-assembly-resources</artifactId>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
</project>
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
<assembly
|
||||||
|
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||||
|
|
||||||
|
<id>oozie-installer</id>
|
||||||
|
<formats>
|
||||||
|
<format>dir</format>
|
||||||
|
</formats>
|
||||||
|
|
||||||
|
<fileSets>
|
||||||
|
<fileSet>
|
||||||
|
<filtered>true</filtered>
|
||||||
|
<directory>${project.build.directory}/assembly-resources/commands</directory>
|
||||||
|
<!--
|
||||||
|
dziala dla (lokalnie zasoby modulu):
|
||||||
|
<directory>src/main/resources</directory>
|
||||||
|
nie dziala dla:
|
||||||
|
<directory>classpath:/commands</directory>
|
||||||
|
<directory>commands</directory>
|
||||||
|
<directory>classpath/src/main/resources</directory>
|
||||||
|
-->
|
||||||
|
<outputDirectory>/</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*</include>
|
||||||
|
</includes>
|
||||||
|
<fileMode>0755</fileMode>
|
||||||
|
<lineEnding>unix</lineEnding>
|
||||||
|
</fileSet>
|
||||||
|
</fileSets>
|
||||||
|
<baseDirectory>/</baseDirectory>
|
||||||
|
</assembly>
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
<assembly
|
||||||
|
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||||
|
|
||||||
|
<id>tests</id>
|
||||||
|
<formats>
|
||||||
|
<format>jar</format>
|
||||||
|
</formats>
|
||||||
|
<includeBaseDirectory>false</includeBaseDirectory>
|
||||||
|
<fileSets>
|
||||||
|
<fileSet>
|
||||||
|
<directory>${project.build.testOutputDirectory}
|
||||||
|
</directory>
|
||||||
|
<outputDirectory />
|
||||||
|
</fileSet>
|
||||||
|
</fileSets>
|
||||||
|
<!-- <dependencySets>
|
||||||
|
<dependencySet>
|
||||||
|
<useProjectArtifact>false</useProjectArtifact>
|
||||||
|
<outputDirectory>lib</outputDirectory>
|
||||||
|
</dependencySet>
|
||||||
|
</dependencySets>-->
|
||||||
|
</assembly>
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
#!/bin/bash
|
||||||
|
hadoop fs -get ${workingDir}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
#!/bin/bash
|
||||||
|
echo ""
|
||||||
|
echo "---->Contents of the working directory"
|
||||||
|
hadoop fs -ls ${workingDir}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
Execute the scripts in the following order:
|
||||||
|
|
||||||
|
1. `upload_workflow.sh`
|
||||||
|
2. `run_workflow.sh`
|
||||||
|
3. `print_working_dir.sh` or `get_working_dir.sh`
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
if [ $# = 0 ] ; then
|
||||||
|
oozie job -oozie ${oozieServiceLoc} -config job.properties -run
|
||||||
|
else
|
||||||
|
oozie job -oozie ${oozieServiceLoc} -config $1/job.properties -run
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
#!/bin/bash
|
||||||
|
exec 3>&1
|
||||||
|
BASH_XTRACEFD=3
|
||||||
|
set -x ## print every executed command
|
||||||
|
|
||||||
|
|
||||||
|
if [ $# = 0 ] ; then
|
||||||
|
target_dir_root=`pwd`'/${oozieAppDir}'
|
||||||
|
else
|
||||||
|
target_dir_root=`readlink -f $1`'/${oozieAppDir}'
|
||||||
|
fi
|
||||||
|
|
||||||
|
# initial phase, creating symbolic links to jars in all subworkflows
|
||||||
|
# currently disabled
|
||||||
|
#libDir=$target_dir_root'/lib'
|
||||||
|
#dirs=`find $target_dir_root/* -maxdepth 10 -type d`
|
||||||
|
#for dir in $dirs
|
||||||
|
#do
|
||||||
|
# if [ -f $dir/workflow.xml ]
|
||||||
|
# then
|
||||||
|
# echo "creating symbolic links to jars in directory: $dir/lib"
|
||||||
|
# if [ ! -d "$dir/lib" ]; then
|
||||||
|
# mkdir $dir/lib
|
||||||
|
# fi
|
||||||
|
# find $libDir -type f -exec ln -s \{\} $dir/lib \;
|
||||||
|
# fi
|
||||||
|
#done
|
||||||
|
|
||||||
|
|
||||||
|
#uploading
|
||||||
|
hadoop fs -rm -r ${sandboxDir}
|
||||||
|
hadoop fs -mkdir -p ${sandboxDir}
|
||||||
|
hadoop fs -mkdir -p ${workingDir}
|
||||||
|
hadoop fs -put $target_dir_root ${sandboxDir}
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
#sandboxName when not provided explicitly will be generated
|
||||||
|
sandboxName=${sandboxName}
|
||||||
|
sandboxDir=/user/${iis.hadoop.frontend.user.name}/${sandboxName}
|
||||||
|
workingDir=${sandboxDir}/working_dir
|
||||||
|
oozie.wf.application.path = ${nameNode}${sandboxDir}/${oozieAppDir}
|
||||||
|
oozieTopWfApplicationPath = ${oozie.wf.application.path}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
Maven plugin module utilized by `dhp-workflows` for proper `job.properties` file building.
|
||||||
|
|
||||||
|
It is based on http://site.kuali.org/maven/plugins/properties-maven-plugin/1.3.2/write-project-properties-mojo.html and supplemented with:
|
||||||
|
|
||||||
|
* handling includePropertyKeysFromFiles property allowing writing only properties listed in given property files
|
||||||
|
As a final outcome only properties listed in `<include>` element and listed as a keys in files from `<includePropertyKeysFromFiles>` element will be written to output file.
|
||||||
|
|
@ -0,0 +1,81 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||||
|
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<parent>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>dhp-build</artifactId>
|
||||||
|
<version>1.0.0-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<artifactId>dhp-build-properties-maven-plugin</artifactId>
|
||||||
|
<packaging>maven-plugin</packaging>
|
||||||
|
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.maven</groupId>
|
||||||
|
<artifactId>maven-plugin-api</artifactId>
|
||||||
|
<version>2.0</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.maven</groupId>
|
||||||
|
<artifactId>maven-project</artifactId>
|
||||||
|
<version>2.0</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.kuali.maven.plugins</groupId>
|
||||||
|
<artifactId>properties-maven-plugin</artifactId>
|
||||||
|
<version>1.3.2</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.google.code.findbugs</groupId>
|
||||||
|
<artifactId>annotations</artifactId>
|
||||||
|
<version>3.0.1</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.google.code.findbugs</groupId>
|
||||||
|
<artifactId>jsr305</artifactId>
|
||||||
|
<version>3.0.1</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<directory>target</directory>
|
||||||
|
<outputDirectory>target/classes</outputDirectory>
|
||||||
|
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||||
|
<testOutputDirectory>target/test-classes</testOutputDirectory>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-source-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>attach-sources</id>
|
||||||
|
<phase>verify</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>jar-no-fork</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-javadoc-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<detectLinks>true</detectLinks>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
</project>
|
||||||
|
|
@ -0,0 +1,71 @@
|
||||||
|
package eu.dnetlib.maven.plugin.properties;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.ArrayUtils;
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
import org.apache.maven.plugin.AbstractMojo;
|
||||||
|
import org.apache.maven.plugin.MojoExecutionException;
|
||||||
|
import org.apache.maven.plugin.MojoFailureException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates oozie properties which were not provided from commandline.
|
||||||
|
* @author mhorst
|
||||||
|
*
|
||||||
|
* @goal generate-properties
|
||||||
|
*/
|
||||||
|
public class GenerateOoziePropertiesMojo extends AbstractMojo {
|
||||||
|
|
||||||
|
public static final String PROPERTY_NAME_WF_SOURCE_DIR = "workflow.source.dir";
|
||||||
|
public static final String PROPERTY_NAME_SANDBOX_NAME = "sandboxName";
|
||||||
|
|
||||||
|
private final String[] limiters = {"iis", "dnetlib", "eu"};
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void execute() throws MojoExecutionException, MojoFailureException {
|
||||||
|
if (System.getProperties().containsKey(PROPERTY_NAME_WF_SOURCE_DIR) &&
|
||||||
|
!System.getProperties().containsKey(PROPERTY_NAME_SANDBOX_NAME)) {
|
||||||
|
String generatedSandboxName = generateSandboxName(System.getProperties().getProperty(
|
||||||
|
PROPERTY_NAME_WF_SOURCE_DIR));
|
||||||
|
if (generatedSandboxName!=null) {
|
||||||
|
System.getProperties().setProperty(PROPERTY_NAME_SANDBOX_NAME,
|
||||||
|
generatedSandboxName);
|
||||||
|
} else {
|
||||||
|
System.out.println("unable to generate sandbox name from path: " +
|
||||||
|
System.getProperties().getProperty(PROPERTY_NAME_WF_SOURCE_DIR));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates sandbox name from workflow source directory.
|
||||||
|
* @param wfSourceDir
|
||||||
|
* @return generated sandbox name
|
||||||
|
*/
|
||||||
|
private String generateSandboxName(String wfSourceDir) {
|
||||||
|
// utilize all dir names until finding one of the limiters
|
||||||
|
List<String> sandboxNameParts = new ArrayList<String>();
|
||||||
|
String[] tokens = StringUtils.split(wfSourceDir, File.separatorChar);
|
||||||
|
ArrayUtils.reverse(tokens);
|
||||||
|
if (tokens.length>0) {
|
||||||
|
for (String token : tokens) {
|
||||||
|
for (String limiter : limiters) {
|
||||||
|
if (limiter.equals(token)) {
|
||||||
|
return sandboxNameParts.size()>0?
|
||||||
|
StringUtils.join(sandboxNameParts.toArray()):null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (sandboxNameParts.size()>0) {
|
||||||
|
sandboxNameParts.add(0, File.separator);
|
||||||
|
}
|
||||||
|
sandboxNameParts.add(0, token);
|
||||||
|
}
|
||||||
|
return StringUtils.join(sandboxNameParts.toArray());
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,436 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Licensed under the Educational Community License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.opensource.org/licenses/ecl2.php
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package eu.dnetlib.maven.plugin.properties;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
import java.util.Properties;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
import org.apache.maven.plugin.AbstractMojo;
|
||||||
|
import org.apache.maven.plugin.MojoExecutionException;
|
||||||
|
import org.apache.maven.plugin.MojoFailureException;
|
||||||
|
import org.apache.maven.project.MavenProject;
|
||||||
|
import org.springframework.core.io.DefaultResourceLoader;
|
||||||
|
import org.springframework.core.io.Resource;
|
||||||
|
import org.springframework.core.io.ResourceLoader;
|
||||||
|
|
||||||
|
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Writes project properties for the keys listed in specified properties files.
|
||||||
|
* Based on:
|
||||||
|
* http://site.kuali.org/maven/plugins/properties-maven-plugin/1.3.2/write-project-properties-mojo.html
|
||||||
|
|
||||||
|
* @author mhorst
|
||||||
|
* @goal write-project-properties
|
||||||
|
*/
|
||||||
|
public class WritePredefinedProjectProperties extends AbstractMojo {
|
||||||
|
|
||||||
|
private static final String CR = "\r";
|
||||||
|
private static final String LF = "\n";
|
||||||
|
private static final String TAB = "\t";
|
||||||
|
protected static final String PROPERTY_PREFIX_ENV = "env.";
|
||||||
|
private static final String ENCODING_UTF8 = "utf8";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @parameter property="properties.includePropertyKeysFromFiles"
|
||||||
|
*/
|
||||||
|
private String[] includePropertyKeysFromFiles;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @parameter default-value="${project}"
|
||||||
|
* @required
|
||||||
|
* @readonly
|
||||||
|
*/
|
||||||
|
protected MavenProject project;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The file that properties will be written to
|
||||||
|
*
|
||||||
|
* @parameter property="properties.outputFile"
|
||||||
|
* default-value="${project.build.directory}/properties/project.properties";
|
||||||
|
* @required
|
||||||
|
*/
|
||||||
|
protected File outputFile;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If true, the plugin will silently ignore any non-existent properties files, and the build will continue
|
||||||
|
*
|
||||||
|
* @parameter property="properties.quiet" default-value="true"
|
||||||
|
*/
|
||||||
|
private boolean quiet;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Comma separated list of characters to escape when writing property values. cr=carriage return, lf=linefeed,
|
||||||
|
* tab=tab. Any other values are taken literally.
|
||||||
|
*
|
||||||
|
* @parameter default-value="cr,lf,tab" property="properties.escapeChars"
|
||||||
|
*/
|
||||||
|
private String escapeChars;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If true, the plugin will include system properties when writing the properties file. System properties override
|
||||||
|
* both environment variables and project properties.
|
||||||
|
*
|
||||||
|
* @parameter default-value="false" property="properties.includeSystemProperties"
|
||||||
|
*/
|
||||||
|
private boolean includeSystemProperties;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If true, the plugin will include environment variables when writing the properties file. Environment variables
|
||||||
|
* are prefixed with "env". Environment variables override project properties.
|
||||||
|
*
|
||||||
|
* @parameter default-value="false" property="properties.includeEnvironmentVariables"
|
||||||
|
*/
|
||||||
|
private boolean includeEnvironmentVariables;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Comma separated set of properties to exclude when writing the properties file
|
||||||
|
*
|
||||||
|
* @parameter property="properties.exclude"
|
||||||
|
*/
|
||||||
|
private String exclude;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Comma separated set of properties to write to the properties file. If provided, only the properties matching
|
||||||
|
* those supplied here will be written to the properties file.
|
||||||
|
*
|
||||||
|
* @parameter property="properties.include"
|
||||||
|
*/
|
||||||
|
private String include;
|
||||||
|
|
||||||
|
/* (non-Javadoc)
|
||||||
|
* @see org.apache.maven.plugin.AbstractMojo#execute()
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
@SuppressFBWarnings({"NP_UNWRITTEN_FIELD","UWF_UNWRITTEN_FIELD"})
|
||||||
|
public void execute() throws MojoExecutionException, MojoFailureException {
|
||||||
|
Properties properties = new Properties();
|
||||||
|
// Add project properties
|
||||||
|
properties.putAll(project.getProperties());
|
||||||
|
if (includeEnvironmentVariables) {
|
||||||
|
// Add environment variables, overriding any existing properties with the same key
|
||||||
|
properties.putAll(getEnvironmentVariables());
|
||||||
|
}
|
||||||
|
if (includeSystemProperties) {
|
||||||
|
// Add system properties, overriding any existing properties with the same key
|
||||||
|
properties.putAll(System.getProperties());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove properties as appropriate
|
||||||
|
trim(properties, exclude, include);
|
||||||
|
|
||||||
|
String comment = "# " + new Date() + "\n";
|
||||||
|
List<String> escapeTokens = getEscapeChars(escapeChars);
|
||||||
|
|
||||||
|
getLog().info("Creating " + outputFile);
|
||||||
|
writeProperties(outputFile, comment, properties, escapeTokens);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides environment variables.
|
||||||
|
* @return environment variables
|
||||||
|
*/
|
||||||
|
protected static Properties getEnvironmentVariables() {
|
||||||
|
Properties props = new Properties();
|
||||||
|
for (Entry<String, String> entry : System.getenv().entrySet()) {
|
||||||
|
props.setProperty(PROPERTY_PREFIX_ENV + entry.getKey(), entry.getValue());
|
||||||
|
}
|
||||||
|
return props;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Removes properties which should not be written.
|
||||||
|
* @param properties
|
||||||
|
* @param omitCSV
|
||||||
|
* @param includeCSV
|
||||||
|
* @throws MojoExecutionException
|
||||||
|
*/
|
||||||
|
protected void trim(Properties properties, String omitCSV, String includeCSV) throws MojoExecutionException {
|
||||||
|
List<String> omitKeys = getListFromCSV(omitCSV);
|
||||||
|
for (String key : omitKeys) {
|
||||||
|
properties.remove(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> includeKeys = getListFromCSV(includeCSV);
|
||||||
|
// mh: including keys from predefined properties
|
||||||
|
if (includePropertyKeysFromFiles!=null && includePropertyKeysFromFiles.length>0) {
|
||||||
|
for (String currentIncludeLoc : includePropertyKeysFromFiles) {
|
||||||
|
if (validate(currentIncludeLoc)) {
|
||||||
|
Properties p = getProperties(currentIncludeLoc);
|
||||||
|
for (String key : p.stringPropertyNames()) {
|
||||||
|
includeKeys.add(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (includeKeys!=null && !includeKeys.isEmpty()) {
|
||||||
|
// removing only when include keys provided
|
||||||
|
Set<String> keys = properties.stringPropertyNames();
|
||||||
|
for (String key : keys) {
|
||||||
|
if (!includeKeys.contains(key)) {
|
||||||
|
properties.remove(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks whether file exists.
|
||||||
|
* @param location
|
||||||
|
* @return true when exists, false otherwise.
|
||||||
|
*/
|
||||||
|
protected boolean exists(String location) {
|
||||||
|
if (StringUtils.isBlank(location)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
File file = new File(location);
|
||||||
|
if (file.exists()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
ResourceLoader loader = new DefaultResourceLoader();
|
||||||
|
Resource resource = loader.getResource(location);
|
||||||
|
return resource.exists();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validates resource location.
|
||||||
|
* @param location
|
||||||
|
* @return true when valid, false otherwise
|
||||||
|
* @throws MojoExecutionException
|
||||||
|
*/
|
||||||
|
protected boolean validate(String location) throws MojoExecutionException {
|
||||||
|
boolean exists = exists(location);
|
||||||
|
if (exists) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (quiet) {
|
||||||
|
getLog().info("Ignoring non-existent properties file '" + location + "'");
|
||||||
|
return false;
|
||||||
|
} else {
|
||||||
|
throw new MojoExecutionException("Non-existent properties file '" + location + "'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides input stream.
|
||||||
|
* @param location
|
||||||
|
* @return input stream
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
protected InputStream getInputStream(String location) throws IOException {
|
||||||
|
File file = new File(location);
|
||||||
|
if (file.exists()) {
|
||||||
|
return new FileInputStream(location);
|
||||||
|
}
|
||||||
|
ResourceLoader loader = new DefaultResourceLoader();
|
||||||
|
Resource resource = loader.getResource(location);
|
||||||
|
return resource.getInputStream();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates properties for given location.
|
||||||
|
* @param location
|
||||||
|
* @return properties for given location
|
||||||
|
* @throws MojoExecutionException
|
||||||
|
*/
|
||||||
|
protected Properties getProperties(String location) throws MojoExecutionException {
|
||||||
|
InputStream in = null;
|
||||||
|
try {
|
||||||
|
Properties properties = new Properties();
|
||||||
|
in = getInputStream(location);
|
||||||
|
if (location.toLowerCase().endsWith(".xml")) {
|
||||||
|
properties.loadFromXML(in);
|
||||||
|
} else {
|
||||||
|
properties.load(in);
|
||||||
|
}
|
||||||
|
return properties;
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new MojoExecutionException("Error reading properties file " + location, e);
|
||||||
|
} finally {
|
||||||
|
IOUtils.closeQuietly(in);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides escape characters.
|
||||||
|
* @param escapeChars
|
||||||
|
* @return escape characters
|
||||||
|
*/
|
||||||
|
protected List<String> getEscapeChars(String escapeChars) {
|
||||||
|
List<String> tokens = getListFromCSV(escapeChars);
|
||||||
|
List<String> realTokens = new ArrayList<String>();
|
||||||
|
for (String token : tokens) {
|
||||||
|
String realToken = getRealToken(token);
|
||||||
|
realTokens.add(realToken);
|
||||||
|
}
|
||||||
|
return realTokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides real token.
|
||||||
|
* @param token
|
||||||
|
* @return real token
|
||||||
|
*/
|
||||||
|
protected String getRealToken(String token) {
|
||||||
|
if (token.equalsIgnoreCase("CR")) {
|
||||||
|
return CR;
|
||||||
|
} else if (token.equalsIgnoreCase("LF")) {
|
||||||
|
return LF;
|
||||||
|
} else if (token.equalsIgnoreCase("TAB")) {
|
||||||
|
return TAB;
|
||||||
|
} else {
|
||||||
|
return token;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns content.
|
||||||
|
* @param comment
|
||||||
|
* @param properties
|
||||||
|
* @param escapeTokens
|
||||||
|
* @return content
|
||||||
|
*/
|
||||||
|
protected String getContent(String comment, Properties properties, List<String> escapeTokens) {
|
||||||
|
List<String> names = new ArrayList<String>(properties.stringPropertyNames());
|
||||||
|
Collections.sort(names);
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
if (!StringUtils.isBlank(comment)) {
|
||||||
|
sb.append(comment);
|
||||||
|
}
|
||||||
|
for (String name : names) {
|
||||||
|
String value = properties.getProperty(name);
|
||||||
|
String escapedValue = escape(value, escapeTokens);
|
||||||
|
sb.append(name + "=" + escapedValue + "\n");
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Writes properties to given file.
|
||||||
|
* @param file
|
||||||
|
* @param comment
|
||||||
|
* @param properties
|
||||||
|
* @param escapeTokens
|
||||||
|
* @throws MojoExecutionException
|
||||||
|
*/
|
||||||
|
protected void writeProperties(File file, String comment, Properties properties, List<String> escapeTokens)
|
||||||
|
throws MojoExecutionException {
|
||||||
|
try {
|
||||||
|
String content = getContent(comment, properties, escapeTokens);
|
||||||
|
FileUtils.writeStringToFile(file, content, ENCODING_UTF8);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new MojoExecutionException("Error creating properties file", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Escapes characters.
|
||||||
|
* @param s
|
||||||
|
* @param escapeChars
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
protected String escape(String s, List<String> escapeChars) {
|
||||||
|
String result = s;
|
||||||
|
for (String escapeChar : escapeChars) {
|
||||||
|
result = result.replace(escapeChar, getReplacementToken(escapeChar));
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides replacement token.
|
||||||
|
* @param escapeChar
|
||||||
|
* @return replacement token
|
||||||
|
*/
|
||||||
|
protected String getReplacementToken(String escapeChar) {
|
||||||
|
if (escapeChar.equals(CR)) {
|
||||||
|
return "\\r";
|
||||||
|
} else if (escapeChar.equals(LF)) {
|
||||||
|
return "\\n";
|
||||||
|
} else if (escapeChar.equals(TAB)) {
|
||||||
|
return "\\t";
|
||||||
|
} else {
|
||||||
|
return "\\" + escapeChar;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns list from csv.
|
||||||
|
* @param csv
|
||||||
|
* @return list of values generated from CSV
|
||||||
|
*/
|
||||||
|
protected static final List<String> getListFromCSV(String csv) {
|
||||||
|
if (StringUtils.isBlank(csv)) {
|
||||||
|
return new ArrayList<String>();
|
||||||
|
}
|
||||||
|
List<String> list = new ArrayList<String>();
|
||||||
|
String[] tokens = StringUtils.split(csv, ",");
|
||||||
|
for (String token : tokens) {
|
||||||
|
list.add(token.trim());
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIncludeSystemProperties(boolean includeSystemProperties) {
|
||||||
|
this.includeSystemProperties = includeSystemProperties;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setEscapeChars(String escapeChars) {
|
||||||
|
this.escapeChars = escapeChars;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIncludeEnvironmentVariables(boolean includeEnvironmentVariables) {
|
||||||
|
this.includeEnvironmentVariables = includeEnvironmentVariables;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setExclude(String exclude) {
|
||||||
|
this.exclude = exclude;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setInclude(String include) {
|
||||||
|
this.include = include;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setQuiet(boolean quiet) {
|
||||||
|
this.quiet = quiet;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets property files for which keys properties should be included.
|
||||||
|
* @param includePropertyKeysFromFiles
|
||||||
|
*/
|
||||||
|
public void setIncludePropertyKeysFromFiles(
|
||||||
|
String[] includePropertyKeysFromFiles) {
|
||||||
|
if (includePropertyKeysFromFiles!=null) {
|
||||||
|
this.includePropertyKeysFromFiles = Arrays.copyOf(
|
||||||
|
includePropertyKeysFromFiles,
|
||||||
|
includePropertyKeysFromFiles.length);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,101 @@
|
||||||
|
package eu.dnetlib.maven.plugin.properties;
|
||||||
|
|
||||||
|
import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_SANDBOX_NAME;
|
||||||
|
import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_WF_SOURCE_DIR;
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertNull;
|
||||||
|
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author mhorst
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class GenerateOoziePropertiesMojoTest {
|
||||||
|
|
||||||
|
private GenerateOoziePropertiesMojo mojo = new GenerateOoziePropertiesMojo();
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void clearSystemProperties() {
|
||||||
|
System.clearProperty(PROPERTY_NAME_SANDBOX_NAME);
|
||||||
|
System.clearProperty(PROPERTY_NAME_WF_SOURCE_DIR);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteEmpty() throws Exception {
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteSandboxNameAlreadySet() throws Exception {
|
||||||
|
// given
|
||||||
|
String workflowSourceDir = "eu/dnetlib/iis/wf/transformers";
|
||||||
|
String sandboxName = "originalSandboxName";
|
||||||
|
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||||
|
System.setProperty(PROPERTY_NAME_SANDBOX_NAME, sandboxName);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertEquals(sandboxName, System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteEmptyWorkflowSourceDir() throws Exception {
|
||||||
|
// given
|
||||||
|
String workflowSourceDir = "";
|
||||||
|
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteNullSandboxNameGenerated() throws Exception {
|
||||||
|
// given
|
||||||
|
String workflowSourceDir = "eu/dnetlib/iis/";
|
||||||
|
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecute() throws Exception {
|
||||||
|
// given
|
||||||
|
String workflowSourceDir = "eu/dnetlib/iis/wf/transformers";
|
||||||
|
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertEquals("wf/transformers", System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteWithoutRoot() throws Exception {
|
||||||
|
// given
|
||||||
|
String workflowSourceDir = "wf/transformers";
|
||||||
|
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertEquals("wf/transformers", System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,365 @@
|
||||||
|
package eu.dnetlib.maven.plugin.properties;
|
||||||
|
|
||||||
|
import static eu.dnetlib.maven.plugin.properties.WritePredefinedProjectProperties.PROPERTY_PREFIX_ENV;
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertFalse;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
import static org.mockito.Mockito.doReturn;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.FileWriter;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.apache.maven.plugin.MojoExecutionException;
|
||||||
|
import org.apache.maven.project.MavenProject;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.TemporaryFolder;
|
||||||
|
import org.junit.runner.RunWith;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.runners.MockitoJUnitRunner;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author mhorst
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
@RunWith(MockitoJUnitRunner.class)
|
||||||
|
public class WritePredefinedProjectPropertiesTest {
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public TemporaryFolder testFolder = new TemporaryFolder();
|
||||||
|
|
||||||
|
@Mock
|
||||||
|
private MavenProject mavenProject;
|
||||||
|
|
||||||
|
private WritePredefinedProjectProperties mojo;
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void init() {
|
||||||
|
mojo = new WritePredefinedProjectProperties();
|
||||||
|
mojo.outputFile = getPropertiesFileLocation();
|
||||||
|
mojo.project = mavenProject;
|
||||||
|
doReturn(new Properties()).when(mavenProject).getProperties();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------------- TESTS ---------------------------------------------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteEmpty() throws Exception {
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertEquals(0, storedProperties.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteWithProjectProperties() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "projectPropertyKey";
|
||||||
|
String value = "projectPropertyValue";
|
||||||
|
Properties projectProperties = new Properties();
|
||||||
|
projectProperties.setProperty(key, value);
|
||||||
|
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertEquals(1, storedProperties.size());
|
||||||
|
assertTrue(storedProperties.containsKey(key));
|
||||||
|
assertEquals(value, storedProperties.getProperty(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(expected=MojoExecutionException.class)
|
||||||
|
public void testExecuteWithProjectPropertiesAndInvalidOutputFile() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "projectPropertyKey";
|
||||||
|
String value = "projectPropertyValue";
|
||||||
|
Properties projectProperties = new Properties();
|
||||||
|
projectProperties.setProperty(key, value);
|
||||||
|
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||||
|
mojo.outputFile = testFolder.getRoot();
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteWithProjectPropertiesExclusion() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "projectPropertyKey";
|
||||||
|
String value = "projectPropertyValue";
|
||||||
|
String excludedKey = "excludedPropertyKey";
|
||||||
|
String excludedValue = "excludedPropertyValue";
|
||||||
|
Properties projectProperties = new Properties();
|
||||||
|
projectProperties.setProperty(key, value);
|
||||||
|
projectProperties.setProperty(excludedKey, excludedValue);
|
||||||
|
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||||
|
mojo.setExclude(excludedKey);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertEquals(1, storedProperties.size());
|
||||||
|
assertTrue(storedProperties.containsKey(key));
|
||||||
|
assertEquals(value, storedProperties.getProperty(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteWithProjectPropertiesInclusion() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "projectPropertyKey";
|
||||||
|
String value = "projectPropertyValue";
|
||||||
|
String includedKey = "includedPropertyKey";
|
||||||
|
String includedValue = "includedPropertyValue";
|
||||||
|
Properties projectProperties = new Properties();
|
||||||
|
projectProperties.setProperty(key, value);
|
||||||
|
projectProperties.setProperty(includedKey, includedValue);
|
||||||
|
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||||
|
mojo.setInclude(includedKey);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertEquals(1, storedProperties.size());
|
||||||
|
assertTrue(storedProperties.containsKey(includedKey));
|
||||||
|
assertEquals(includedValue, storedProperties.getProperty(includedKey));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteIncludingPropertyKeysFromFile() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "projectPropertyKey";
|
||||||
|
String value = "projectPropertyValue";
|
||||||
|
String includedKey = "includedPropertyKey";
|
||||||
|
String includedValue = "includedPropertyValue";
|
||||||
|
Properties projectProperties = new Properties();
|
||||||
|
projectProperties.setProperty(key, value);
|
||||||
|
projectProperties.setProperty(includedKey, includedValue);
|
||||||
|
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||||
|
|
||||||
|
File includedPropertiesFile = new File(testFolder.getRoot(), "included.properties");
|
||||||
|
Properties includedProperties = new Properties();
|
||||||
|
includedProperties.setProperty(includedKey, "irrelevantValue");
|
||||||
|
includedProperties.store(new FileWriter(includedPropertiesFile), null);
|
||||||
|
|
||||||
|
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertEquals(1, storedProperties.size());
|
||||||
|
assertTrue(storedProperties.containsKey(includedKey));
|
||||||
|
assertEquals(includedValue, storedProperties.getProperty(includedKey));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteIncludingPropertyKeysFromClasspathResource() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "projectPropertyKey";
|
||||||
|
String value = "projectPropertyValue";
|
||||||
|
String includedKey = "includedPropertyKey";
|
||||||
|
String includedValue = "includedPropertyValue";
|
||||||
|
Properties projectProperties = new Properties();
|
||||||
|
projectProperties.setProperty(key, value);
|
||||||
|
projectProperties.setProperty(includedKey, includedValue);
|
||||||
|
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||||
|
|
||||||
|
mojo.setIncludePropertyKeysFromFiles(new String[] {"/eu/dnetlib/maven/plugin/properties/included.properties"});
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertEquals(1, storedProperties.size());
|
||||||
|
assertTrue(storedProperties.containsKey(includedKey));
|
||||||
|
assertEquals(includedValue, storedProperties.getProperty(includedKey));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(expected=MojoExecutionException.class)
|
||||||
|
public void testExecuteIncludingPropertyKeysFromBlankLocation() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "projectPropertyKey";
|
||||||
|
String value = "projectPropertyValue";
|
||||||
|
String includedKey = "includedPropertyKey";
|
||||||
|
String includedValue = "includedPropertyValue";
|
||||||
|
Properties projectProperties = new Properties();
|
||||||
|
projectProperties.setProperty(key, value);
|
||||||
|
projectProperties.setProperty(includedKey, includedValue);
|
||||||
|
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||||
|
|
||||||
|
mojo.setIncludePropertyKeysFromFiles(new String[] {""});
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteIncludingPropertyKeysFromXmlFile() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "projectPropertyKey";
|
||||||
|
String value = "projectPropertyValue";
|
||||||
|
String includedKey = "includedPropertyKey";
|
||||||
|
String includedValue = "includedPropertyValue";
|
||||||
|
Properties projectProperties = new Properties();
|
||||||
|
projectProperties.setProperty(key, value);
|
||||||
|
projectProperties.setProperty(includedKey, includedValue);
|
||||||
|
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||||
|
|
||||||
|
File includedPropertiesFile = new File(testFolder.getRoot(), "included.xml");
|
||||||
|
Properties includedProperties = new Properties();
|
||||||
|
includedProperties.setProperty(includedKey, "irrelevantValue");
|
||||||
|
includedProperties.storeToXML(new FileOutputStream(includedPropertiesFile), null);
|
||||||
|
|
||||||
|
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertEquals(1, storedProperties.size());
|
||||||
|
assertTrue(storedProperties.containsKey(includedKey));
|
||||||
|
assertEquals(includedValue, storedProperties.getProperty(includedKey));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(expected=MojoExecutionException.class)
|
||||||
|
public void testExecuteIncludingPropertyKeysFromInvalidXmlFile() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "projectPropertyKey";
|
||||||
|
String value = "projectPropertyValue";
|
||||||
|
String includedKey = "includedPropertyKey";
|
||||||
|
String includedValue = "includedPropertyValue";
|
||||||
|
Properties projectProperties = new Properties();
|
||||||
|
projectProperties.setProperty(key, value);
|
||||||
|
projectProperties.setProperty(includedKey, includedValue);
|
||||||
|
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||||
|
|
||||||
|
File includedPropertiesFile = new File(testFolder.getRoot(), "included.xml");
|
||||||
|
Properties includedProperties = new Properties();
|
||||||
|
includedProperties.setProperty(includedKey, "irrelevantValue");
|
||||||
|
includedProperties.store(new FileOutputStream(includedPropertiesFile), null);
|
||||||
|
|
||||||
|
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteWithQuietModeOn() throws Exception {
|
||||||
|
// given
|
||||||
|
mojo.setQuiet(true);
|
||||||
|
mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"});
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertEquals(0, storedProperties.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(expected=MojoExecutionException.class)
|
||||||
|
public void testExecuteIncludingPropertyKeysFromInvalidFile() throws Exception {
|
||||||
|
// given
|
||||||
|
mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"});
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteWithEnvironmentProperties() throws Exception {
|
||||||
|
// given
|
||||||
|
mojo.setIncludeEnvironmentVariables(true);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertTrue(storedProperties.size() > 0);
|
||||||
|
for (Object currentKey : storedProperties.keySet()) {
|
||||||
|
assertTrue(((String)currentKey).startsWith(PROPERTY_PREFIX_ENV));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteWithSystemProperties() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "systemPropertyKey";
|
||||||
|
String value = "systemPropertyValue";
|
||||||
|
System.setProperty(key, value);
|
||||||
|
mojo.setIncludeSystemProperties(true);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertTrue(storedProperties.size() > 0);
|
||||||
|
assertTrue(storedProperties.containsKey(key));
|
||||||
|
assertEquals(value, storedProperties.getProperty(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecuteWithSystemPropertiesAndEscapeChars() throws Exception {
|
||||||
|
// given
|
||||||
|
String key = "systemPropertyKey ";
|
||||||
|
String value = "systemPropertyValue";
|
||||||
|
System.setProperty(key, value);
|
||||||
|
mojo.setIncludeSystemProperties(true);
|
||||||
|
String escapeChars = "cr,lf,tab,|";
|
||||||
|
mojo.setEscapeChars(escapeChars);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
mojo.execute();
|
||||||
|
|
||||||
|
// assert
|
||||||
|
assertTrue(mojo.outputFile.exists());
|
||||||
|
Properties storedProperties = getStoredProperties();
|
||||||
|
assertTrue(storedProperties.size() > 0);
|
||||||
|
assertFalse(storedProperties.containsKey(key));
|
||||||
|
assertTrue(storedProperties.containsKey(key.trim()));
|
||||||
|
assertEquals(value, storedProperties.getProperty(key.trim()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------------- PRIVATE -------------------------------------------
|
||||||
|
|
||||||
|
private File getPropertiesFileLocation() {
|
||||||
|
return new File(testFolder.getRoot(), "test.properties");
|
||||||
|
}
|
||||||
|
|
||||||
|
private Properties getStoredProperties() throws FileNotFoundException, IOException {
|
||||||
|
Properties properties = new Properties();
|
||||||
|
properties.load(new FileInputStream(getPropertiesFileLocation()));
|
||||||
|
return properties;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
includedPropertyKey=irrelevantValue
|
||||||
|
|
@ -9,7 +9,8 @@
|
||||||
<artifactId>dhp-build</artifactId>
|
<artifactId>dhp-build</artifactId>
|
||||||
<packaging>pom</packaging>
|
<packaging>pom</packaging>
|
||||||
<modules>
|
<modules>
|
||||||
|
<module>dhp-build-assembly-resources</module>
|
||||||
|
<module>dhp-build-properties-maven-plugin</module>
|
||||||
</modules>
|
</modules>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<parent>
|
||||||
|
<artifactId>dhp-workflows</artifactId>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<version>1.0.0-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<artifactId>dhp-distcp</artifactId>
|
||||||
|
|
||||||
|
|
||||||
|
</project>
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>jobTracker</name>
|
||||||
|
<value>yarnRM</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>nameNode</name>
|
||||||
|
<value>hdfs://nameservice1</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>sourceNN</name>
|
||||||
|
<value>webhdfs://namenode2.hadoop.dm.openaire.eu:50071</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>oozie.use.system.libpath</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
|
|
@ -0,0 +1,46 @@
|
||||||
|
<workflow-app name="distcp" xmlns="uri:oozie:workflow:0.5">
|
||||||
|
<parameters>
|
||||||
|
<property>
|
||||||
|
<name>sourceNN</name>
|
||||||
|
<description>the source name node</description>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>sourcePath</name>
|
||||||
|
<description>the source path</description>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>targetPath</name>
|
||||||
|
<description>the target path</description>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hbase_dump_distcp_memory_mb</name>
|
||||||
|
<value>6144</value>
|
||||||
|
<description>memory for distcp action copying InfoSpace dump from remote cluster</description>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>hbase_dump_distcp_num_maps</name>
|
||||||
|
<value>1</value>
|
||||||
|
<description>maximum number of simultaneous copies of InfoSpace dump from remote location</description>
|
||||||
|
</property>
|
||||||
|
</parameters>
|
||||||
|
|
||||||
|
<start to="distcp"/>
|
||||||
|
|
||||||
|
<kill name="Kill">
|
||||||
|
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||||
|
</kill>
|
||||||
|
|
||||||
|
<action name="distcp">
|
||||||
|
<distcp xmlns="uri:oozie:distcp-action:0.2">
|
||||||
|
<arg>-Dmapreduce.map.memory.mb=${hbase_dump_distcp_memory_mb}</arg>
|
||||||
|
<arg>-pb</arg>
|
||||||
|
<arg>-m ${hbase_dump_distcp_num_maps}</arg>
|
||||||
|
<arg>${sourceNN}/${sourcePath}</arg>
|
||||||
|
<arg>${nameNode}/${targetPath}</arg>
|
||||||
|
</distcp>
|
||||||
|
<ok to="End" />
|
||||||
|
<error to="Kill" />
|
||||||
|
</action>
|
||||||
|
|
||||||
|
<end name="End"/>
|
||||||
|
</workflow-app>
|
||||||
|
|
@ -1,16 +1,604 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||||
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>eu.dnetlib.dhp</groupId>
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
<artifactId>dhp</artifactId>
|
<artifactId>dhp</artifactId>
|
||||||
<version>1.0.0-SNAPSHOT</version>
|
<version>1.0.0-SNAPSHOT</version>
|
||||||
<relativePath>../</relativePath>
|
<relativePath>../</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<artifactId>dhp-workflows</artifactId>
|
<artifactId>dhp-workflows</artifactId>
|
||||||
<packaging>pom</packaging>
|
<packaging>pom</packaging>
|
||||||
|
|
||||||
<modules>
|
<modules>
|
||||||
<module>dhp-aggregation</module>
|
<module>dhp-aggregation</module>
|
||||||
|
<module>dhp-distcp</module>
|
||||||
</modules>
|
</modules>
|
||||||
|
|
||||||
|
<pluginRepositories>
|
||||||
|
<pluginRepository>
|
||||||
|
<id>iis-releases</id>
|
||||||
|
<name>iis releases plugin repository</name>
|
||||||
|
<url>http://maven.ceon.pl/artifactory/iis-releases</url>
|
||||||
|
<layout>default</layout>
|
||||||
|
</pluginRepository>
|
||||||
|
</pluginRepositories>
|
||||||
|
<properties>
|
||||||
|
<maven.build.timestamp.format>yyyy-MM-dd_HH_mm</maven.build.timestamp.format>
|
||||||
|
<!-- default Oozie installer properties requred to be defined at pom.xml level -->
|
||||||
|
<!-- other project properties are defined in project-default.properties -->
|
||||||
|
<oozie.package.file.name>oozie-package</oozie.package.file.name>
|
||||||
|
<!-- notice: sandboxName is generated based on workflow.source.dir property -->
|
||||||
|
<workflow.source.dir>src/test/resources/define/path/pointing/to/directory/holding/oozie_app</workflow.source.dir>
|
||||||
|
<oozieAppDir>oozie_app</oozieAppDir>
|
||||||
|
<queueName>default</queueName>
|
||||||
|
<importerQueueName>default</importerQueueName>
|
||||||
|
<oozieLauncherQueueName>default</oozieLauncherQueueName>
|
||||||
|
<primed.dir>primed</primed.dir>
|
||||||
|
|
||||||
|
<oozie.package.dependencies.include.scope>runtime</oozie.package.dependencies.include.scope>
|
||||||
|
<oozie.package.dependencies.exclude.scope></oozie.package.dependencies.exclude.scope>
|
||||||
|
<oozie.package.skip.test.jar>true</oozie.package.skip.test.jar>
|
||||||
|
|
||||||
|
<dhpConnectionProperties>${user.home}/.dhp/application.properties</dhpConnectionProperties>
|
||||||
|
|
||||||
|
<output.dir.name>${maven.build.timestamp}</output.dir.name>
|
||||||
|
|
||||||
|
<projectVersion>${project.version}</projectVersion>
|
||||||
|
<oozie.use.system.libpath>true</oozie.use.system.libpath>
|
||||||
|
</properties>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>dhp-build-assembly-resources</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.oozie</groupId>
|
||||||
|
<artifactId>oozie-client</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>net.schmizz</groupId>
|
||||||
|
<artifactId>sshj</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
<profiles>
|
||||||
|
<profile>
|
||||||
|
<!-- This profile sets properties that are required for test oozie workflows To be used only with 'oozie-package' profile -->
|
||||||
|
<id>attach-test-resources</id>
|
||||||
|
<properties>
|
||||||
|
<!--overriding default scope (set to 'runtime') with the 'test' value. Test resources attached to oozie package requires all test dependencies. -->
|
||||||
|
<oozie.package.dependencies.include.scope></oozie.package.dependencies.include.scope>
|
||||||
|
<oozie.package.dependencies.exclude.scope>provided</oozie.package.dependencies.exclude.scope>
|
||||||
|
<!-- Do not skip creation of test jar for priming (in oozie-package profile) -->
|
||||||
|
<oozie.package.skip.test.jar>false</oozie.package.skip.test.jar>
|
||||||
|
</properties>
|
||||||
|
</profile>
|
||||||
|
<profile>
|
||||||
|
<id>oozie-package</id>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-enforcer-plugin</artifactId>
|
||||||
|
<version>1.4.1</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>enforce-connection-properties-file-existence</id>
|
||||||
|
<phase>initialize</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>enforce</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<rules>
|
||||||
|
<requireFilesExist>
|
||||||
|
<files>
|
||||||
|
<file>${dhpConnectionProperties}</file>
|
||||||
|
</files>
|
||||||
|
<message>
|
||||||
|
The file with connection properties could not be found. Please, create the ${dhpConnectionProperties} file or set the location to another already created file by using
|
||||||
|
-DdhpConnectionProperties property.
|
||||||
|
</message>
|
||||||
|
</requireFilesExist>
|
||||||
|
</rules>
|
||||||
|
<fail>true</fail>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-dependency-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>copy dependencies</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>copy-dependencies</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<includeScope>${oozie.package.dependencies.include.scope}</includeScope>
|
||||||
|
<excludeScope>${oozie.package.dependencies.exclude.scope}</excludeScope>
|
||||||
|
<silent>true</silent>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<!-- Plugin originally defined in attach-test-resources It was moved here to ensure that it will execute before priming -->
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-jar-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>attach-test-resources-package</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>test-jar</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<skip>${oozie.package.skip.test.jar}</skip>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
|
||||||
|
<plugin>
|
||||||
|
<groupId>eu.dnetlib.primer</groupId>
|
||||||
|
<artifactId>primer-maven-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>priming</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>prime</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<classProviderFiles>
|
||||||
|
<classProviderFile>${project.build.directory}/dependency/*.jar</classProviderFile>
|
||||||
|
<classProviderFile>${project.build.directory}/*-tests.jar</classProviderFile>
|
||||||
|
<classProviderFile>${project.build.directory}/classes</classProviderFile>
|
||||||
|
</classProviderFiles>
|
||||||
|
<coansysPackageDir>${project.build.directory}/dependency</coansysPackageDir>
|
||||||
|
<destination>${project.build.directory}/${primed.dir}</destination>
|
||||||
|
<classpath>${workflow.source.dir}</classpath>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<!-- reading job.properties to use them in .sh scripts -->
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.kuali.maven.plugins</groupId>
|
||||||
|
<artifactId>properties-maven-plugin</artifactId>
|
||||||
|
<version>1.3.2</version>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>dhp-build-assembly-resources</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<!-- contains project-default.properties -->
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>reading-dhp-properties</id>
|
||||||
|
<phase>initialize</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>read-project-properties</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<locations>
|
||||||
|
<location>${dhpConnectionProperties}</location>
|
||||||
|
</locations>
|
||||||
|
<quiet>false</quiet>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>read-default-properties</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>read-project-properties</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<locations>
|
||||||
|
<location>classpath:project-default.properties</location>
|
||||||
|
</locations>
|
||||||
|
<quiet>true</quiet>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>read-job-properties</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>read-project-properties</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<locations>
|
||||||
|
<param>${project.build.directory}/${primed.dir}/job.properties</param>
|
||||||
|
<param>job-override.properties</param>
|
||||||
|
</locations>
|
||||||
|
<quiet>true</quiet>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>dhp-build-properties-maven-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<phase>validate</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>generate-properties</goal>
|
||||||
|
<!-- generates sandboxName based on workflow.source.dir when not specified as commandline parameter -->
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>write-job-properties</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>write-project-properties</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<outputFile>target/${oozie.package.file.name}/job.properties</outputFile>
|
||||||
|
<!-- notice: dots are not allowed for job.properties! -->
|
||||||
|
<include>nameNode,jobTracker,queueName,importerQueueName,oozieLauncherQueueName,
|
||||||
|
workingDir,oozieTopWfApplicationPath,oozieServiceLoc,
|
||||||
|
sparkDriverMemory,sparkExecutorMemory,sparkExecutorCores,
|
||||||
|
oozie.wf.application.path,projectVersion,oozie.use.system.libpath</include>
|
||||||
|
<includeSystemProperties>true</includeSystemProperties>
|
||||||
|
<includePropertyKeysFromFiles>
|
||||||
|
<!-- <param>${workflow.source.dir}/job.properties</param> -->
|
||||||
|
<param>${project.build.directory}/${primed.dir}/job.properties</param>
|
||||||
|
<param>job-override.properties</param>
|
||||||
|
|
||||||
|
</includePropertyKeysFromFiles>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>pl.project13.maven</groupId>
|
||||||
|
<artifactId>git-commit-id-plugin</artifactId>
|
||||||
|
<version>2.1.11</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<goals>
|
||||||
|
<goal>revision</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<verbose>true</verbose>
|
||||||
|
<dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
|
||||||
|
<generateGitPropertiesFile>true</generateGitPropertiesFile>
|
||||||
|
<generateGitPropertiesFilename>target/${oozie.package.file.name}/${oozieAppDir}/version.properties</generateGitPropertiesFilename>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<version>3.0.0</version>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>dhp-build-assembly-resources</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>assembly-oozie-installer</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<appendAssemblyId>false</appendAssemblyId>
|
||||||
|
<finalName>${oozie.package.file.name}_shell_scripts</finalName>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>oozie-installer</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
|
||||||
|
<plugin>
|
||||||
|
<!-- this plugin prepares oozie installer package-->
|
||||||
|
|
||||||
|
<artifactId>maven-antrun-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<!-- extracting shared resources phase -->
|
||||||
|
<execution>
|
||||||
|
<id>installer-copy-custom</id>
|
||||||
|
<phase>process-resources</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>run</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<tasks>
|
||||||
|
<property name="assembly-resources.loc" value="${maven.dependency.eu.dnetlib.dhp.dhp-build-assembly-resources.jar.path}" />
|
||||||
|
<unjar src="${assembly-resources.loc}" dest="${project.build.directory}/assembly-resources" />
|
||||||
|
</tasks>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<!-- packaging phase -->
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<configuration>
|
||||||
|
<tasks>
|
||||||
|
<!-- copying workflow resources -->
|
||||||
|
<mkdir dir="target/${oozie.package.file.name}" />
|
||||||
|
<mkdir dir="target/${oozie.package.file.name}/${oozieAppDir}" />
|
||||||
|
<copy todir="target/${oozie.package.file.name}/${oozieAppDir}">
|
||||||
|
<!-- <fileset dir="${workflow.source.dir}/${oozieAppDir}" /> replacing with primed dir location -->
|
||||||
|
<fileset dir="target/${primed.dir}/${oozieAppDir}" />
|
||||||
|
</copy>
|
||||||
|
<!-- copying all jars to oozie lib directory -->
|
||||||
|
<mkdir dir="target/${oozie.package.file.name}/${oozieAppDir}/lib" />
|
||||||
|
<copy todir="target/${oozie.package.file.name}/${oozieAppDir}/lib">
|
||||||
|
<fileset dir="${project.build.directory}/dependency" />
|
||||||
|
</copy>
|
||||||
|
<!-- copying current module lib -->
|
||||||
|
<copy todir="target/${oozie.package.file.name}/${oozieAppDir}/lib">
|
||||||
|
<fileset dir="${project.build.directory}">
|
||||||
|
<include name="*.jar" />
|
||||||
|
</fileset>
|
||||||
|
</copy>
|
||||||
|
|
||||||
|
|
||||||
|
<fixcrlf srcdir="target/${oozie.package.file.name}/${oozieAppDir}/" encoding="UTF-8" outputencoding="UTF-8" includes="**/*.sh,**/*.json,**/*.py,**/*.sql" eol="lf"/>
|
||||||
|
|
||||||
|
|
||||||
|
<!-- creating tar.gz package -->
|
||||||
|
<tar destfile="target/${oozie.package.file.name}.tar.gz" compression="gzip">
|
||||||
|
<tarfileset dir="target/${oozie.package.file.name}" />
|
||||||
|
<tarfileset dir="target/${oozie.package.file.name}_shell_scripts" filemode="0755">
|
||||||
|
<include name="**/*.sh" />
|
||||||
|
</tarfileset>
|
||||||
|
<tarfileset dir="target/${oozie.package.file.name}_shell_scripts" filemode="0644">
|
||||||
|
<exclude name="**/*.sh" />
|
||||||
|
</tarfileset>
|
||||||
|
</tar>
|
||||||
|
<!-- cleanup -->
|
||||||
|
<delete dir="target/${oozie.package.file.name}" />
|
||||||
|
<delete dir="target/${oozie.package.file.name}_shell_scripts" />
|
||||||
|
</tasks>
|
||||||
|
</configuration>
|
||||||
|
<goals>
|
||||||
|
<goal>run</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</profile>
|
||||||
|
<profile>
|
||||||
|
<id>deploy</id>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.codehaus.mojo</groupId>
|
||||||
|
<artifactId>exec-maven-plugin</artifactId>
|
||||||
|
<version>1.5.0</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>create-target-dir</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>exec</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<executable>ssh</executable>
|
||||||
|
<arguments>
|
||||||
|
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
|
||||||
|
<argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
|
||||||
|
<argument>-o StrictHostKeyChecking=no</argument>
|
||||||
|
<argument>rm -rf ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; mkdir -p ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/</argument>
|
||||||
|
</arguments>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>upload-oozie-package</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>exec</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<executable>scp</executable>
|
||||||
|
<arguments>
|
||||||
|
<argument>-P ${dhp.hadoop.frontend.port.ssh}</argument>
|
||||||
|
<argument>-o StrictHostKeyChecking=no</argument>
|
||||||
|
<argument>target/${oozie.package.file.name}.tar.gz</argument>
|
||||||
|
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}:${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/${oozie.package.file.name}.tar.gz</argument>
|
||||||
|
</arguments>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>extract-and-upload-to-hdfs</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>exec</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<executable>ssh</executable>
|
||||||
|
<!-- <outputFile>target/redirected_upload.log</outputFile> -->
|
||||||
|
<arguments>
|
||||||
|
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
|
||||||
|
<argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
|
||||||
|
<argument>-o StrictHostKeyChecking=no</argument>
|
||||||
|
<argument>cd ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; </argument>
|
||||||
|
<argument>tar -zxf oozie-package.tar.gz; </argument>
|
||||||
|
<argument>rm ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/oozie-package.tar.gz; </argument>
|
||||||
|
<argument>./upload_workflow.sh</argument>
|
||||||
|
</arguments>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</profile>
|
||||||
|
<profile>
|
||||||
|
<id>run</id>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.codehaus.mojo</groupId>
|
||||||
|
<artifactId>exec-maven-plugin</artifactId>
|
||||||
|
<version>1.5.0</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>run-job</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>exec</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<executable>ssh</executable>
|
||||||
|
<!-- this file will be used by test verification profile reading job identifier -->
|
||||||
|
<outputFile>${oozie.execution.log.file.location}</outputFile>
|
||||||
|
<arguments>
|
||||||
|
<argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
|
||||||
|
<argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
|
||||||
|
<argument>-o StrictHostKeyChecking=no</argument>
|
||||||
|
<argument>cd ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; </argument>
|
||||||
|
<argument>./run_workflow.sh</argument>
|
||||||
|
</arguments>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>show-run-log-on-stdout</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>exec</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<executable>cat</executable>
|
||||||
|
<arguments>
|
||||||
|
<argument>${oozie.execution.log.file.location}</argument>
|
||||||
|
</arguments>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</profile>
|
||||||
|
<profile>
|
||||||
|
<!-- this profile is handling unit and integration test definitions of all child modules -->
|
||||||
|
<id>child-tests</id>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-jar-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>integration-test-package</id>
|
||||||
|
<phase>integration-test</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>test-jar</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-surefire-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<excludedGroups>eu.dnetlib.iis.common.IntegrationTest</excludedGroups>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-failsafe-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>default-integration-test</id>
|
||||||
|
<configuration>
|
||||||
|
<argLine>-Xmx1024m</argLine>
|
||||||
|
<systemPropertiesVariables>
|
||||||
|
<!-- if dhpConnectionProperties is not defined, then ${user.home}/.dhp/integration-test.properties will be used -->
|
||||||
|
<dhpConnectionProperties>${dhpConnectionProperties}</dhpConnectionProperties>
|
||||||
|
<output.dir.name>${output.dir.name}</output.dir.name>
|
||||||
|
</systemPropertiesVariables>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
<activation>
|
||||||
|
<file>
|
||||||
|
<missing>src/main/resources/parent.marker</missing>
|
||||||
|
</file>
|
||||||
|
</activation>
|
||||||
|
</profile>
|
||||||
|
</profiles>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<pluginManagement>
|
||||||
|
<plugins>
|
||||||
|
<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself. -->
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.eclipse.m2e</groupId>
|
||||||
|
<artifactId>lifecycle-mapping</artifactId>
|
||||||
|
<version>1.0.0</version>
|
||||||
|
<configuration>
|
||||||
|
<lifecycleMappingMetadata>
|
||||||
|
<pluginExecutions>
|
||||||
|
<pluginExecution>
|
||||||
|
<pluginExecutionFilter>
|
||||||
|
<groupId>
|
||||||
|
org.kuali.maven.plugins
|
||||||
|
</groupId>
|
||||||
|
<artifactId>
|
||||||
|
properties-maven-plugin
|
||||||
|
</artifactId>
|
||||||
|
<versionRange>
|
||||||
|
[1.3.2,)
|
||||||
|
</versionRange>
|
||||||
|
<goals>
|
||||||
|
<goal>
|
||||||
|
read-project-properties
|
||||||
|
</goal>
|
||||||
|
<goal>
|
||||||
|
write-project-properties
|
||||||
|
</goal>
|
||||||
|
</goals>
|
||||||
|
</pluginExecutionFilter>
|
||||||
|
<action>
|
||||||
|
<ignore />
|
||||||
|
</action>
|
||||||
|
</pluginExecution>
|
||||||
|
<!-- copy-dependency plugin -->
|
||||||
|
<pluginExecution>
|
||||||
|
<pluginExecutionFilter>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-dependency-plugin</artifactId>
|
||||||
|
<versionRange>[1.0.0,)</versionRange>
|
||||||
|
<goals>
|
||||||
|
<goal>copy-dependencies</goal>
|
||||||
|
</goals>
|
||||||
|
</pluginExecutionFilter>
|
||||||
|
<action>
|
||||||
|
<ignore />
|
||||||
|
</action>
|
||||||
|
</pluginExecution>
|
||||||
|
</pluginExecutions>
|
||||||
|
</lifecycleMappingMetadata>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</pluginManagement>
|
||||||
|
</build>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
94
pom.xml
94
pom.xml
|
|
@ -20,7 +20,7 @@
|
||||||
</licenses>
|
</licenses>
|
||||||
|
|
||||||
<modules>
|
<modules>
|
||||||
<!--<module>dhp-build</module>-->
|
<module>dhp-build</module>
|
||||||
<module>dhp-common</module>
|
<module>dhp-common</module>
|
||||||
<module>dhp-workflows</module>
|
<module>dhp-workflows</module>
|
||||||
<module>dhp-applications</module>
|
<module>dhp-applications</module>
|
||||||
|
|
@ -60,6 +60,8 @@
|
||||||
<enabled>true</enabled>
|
<enabled>true</enabled>
|
||||||
</releases>
|
</releases>
|
||||||
</repository>
|
</repository>
|
||||||
|
|
||||||
|
<!--
|
||||||
<repository>
|
<repository>
|
||||||
<id>dnet45-bootstrap-release</id>
|
<id>dnet45-bootstrap-release</id>
|
||||||
<name>dnet45 bootstrap release</name>
|
<name>dnet45 bootstrap release</name>
|
||||||
|
|
@ -72,6 +74,7 @@
|
||||||
<enabled>true</enabled>
|
<enabled>true</enabled>
|
||||||
</releases>
|
</releases>
|
||||||
</repository>
|
</repository>
|
||||||
|
-->
|
||||||
|
|
||||||
<repository>
|
<repository>
|
||||||
<id>cloudera</id>
|
<id>cloudera</id>
|
||||||
|
|
@ -142,6 +145,26 @@
|
||||||
<artifactId>commons-cli</artifactId>
|
<artifactId>commons-cli</artifactId>
|
||||||
<version>1.4</version>
|
<version>1.4</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>net.schmizz</groupId>
|
||||||
|
<artifactId>sshj</artifactId>
|
||||||
|
<version>0.10.0</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.oozie</groupId>
|
||||||
|
<artifactId>oozie-client</artifactId>
|
||||||
|
<version>${dhp.oozie.version}</version>
|
||||||
|
<exclusions>
|
||||||
|
<!-- conflicts -->
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>slf4j-simple</artifactId>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</dependencyManagement>
|
</dependencyManagement>
|
||||||
|
|
||||||
|
|
@ -192,6 +215,30 @@
|
||||||
<redirectTestOutputToFile>true</redirectTestOutputToFile>
|
<redirectTestOutputToFile>true</redirectTestOutputToFile>
|
||||||
</configuration>
|
</configuration>
|
||||||
</plugin>
|
</plugin>
|
||||||
|
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-failsafe-plugin</artifactId>
|
||||||
|
<version>2.19.1</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>default-integration-test</id>
|
||||||
|
<goals>
|
||||||
|
<goal>integration-test</goal>
|
||||||
|
<goal>verify</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<groups>eu.dnetlib.dhp.common.IntegrationTest</groups>
|
||||||
|
<includes>
|
||||||
|
<include>**/*Test.java</include>
|
||||||
|
</includes>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<plugin>
|
<plugin>
|
||||||
<groupId>org.apache.maven.plugins</groupId>
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
<artifactId>maven-javadoc-plugin</artifactId>
|
<artifactId>maven-javadoc-plugin</artifactId>
|
||||||
|
|
@ -205,7 +252,13 @@
|
||||||
<artifactId>maven-dependency-plugin</artifactId>
|
<artifactId>maven-dependency-plugin</artifactId>
|
||||||
<version>3.0.0</version>
|
<version>3.0.0</version>
|
||||||
</plugin>
|
</plugin>
|
||||||
<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself. -->
|
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.codehaus.mojo</groupId>
|
||||||
|
<artifactId>build-helper-maven-plugin</artifactId>
|
||||||
|
<version>1.12</version>
|
||||||
|
</plugin>
|
||||||
|
|
||||||
</plugins>
|
</plugins>
|
||||||
</pluginManagement>
|
</pluginManagement>
|
||||||
|
|
||||||
|
|
@ -215,8 +268,42 @@
|
||||||
<artifactId>maven-release-plugin</artifactId>
|
<artifactId>maven-release-plugin</artifactId>
|
||||||
<version>2.5.3</version>
|
<version>2.5.3</version>
|
||||||
</plugin>
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.jacoco</groupId>
|
||||||
|
<artifactId>jacoco-maven-plugin</artifactId>
|
||||||
|
<version>0.7.9</version>
|
||||||
|
<configuration>
|
||||||
|
<excludes>
|
||||||
|
<exclude>**/schemas/*</exclude>
|
||||||
|
<exclude>**/com/cloudera/**/*</exclude>
|
||||||
|
<exclude>**/org/apache/avro/io/**/*</exclude>
|
||||||
|
</excludes>
|
||||||
|
</configuration>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>default-prepare-agent</id>
|
||||||
|
<goals>
|
||||||
|
<goal>prepare-agent</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>default-report</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>report</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
</plugins>
|
</plugins>
|
||||||
|
|
||||||
|
<extensions>
|
||||||
|
<extension>
|
||||||
|
<groupId>org.apache.maven.wagon</groupId>
|
||||||
|
<artifactId>wagon-ssh</artifactId>
|
||||||
|
<version>2.10</version>
|
||||||
|
</extension>
|
||||||
|
</extensions>
|
||||||
|
|
||||||
</build>
|
</build>
|
||||||
|
|
||||||
|
|
@ -250,9 +337,12 @@
|
||||||
<properties>
|
<properties>
|
||||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||||
|
|
||||||
<dhp.cdh.version>cdh5.9.2</dhp.cdh.version>
|
<dhp.cdh.version>cdh5.9.2</dhp.cdh.version>
|
||||||
<dhp.hadoop.version>2.6.0-${dhp.cdh.version}</dhp.hadoop.version>
|
<dhp.hadoop.version>2.6.0-${dhp.cdh.version}</dhp.hadoop.version>
|
||||||
|
<dhp.oozie.version>4.1.0-${dhp.cdh.version}</dhp.oozie.version>
|
||||||
<dhp.spark.version>2.2.0</dhp.spark.version>
|
<dhp.spark.version>2.2.0</dhp.spark.version>
|
||||||
|
|
||||||
<dhp.commons.lang.version>3.5</dhp.commons.lang.version>
|
<dhp.commons.lang.version>3.5</dhp.commons.lang.version>
|
||||||
<scala.version>2.11.8</scala.version>
|
<scala.version>2.11.8</scala.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue