<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <parent>
        <groupId>eu.dnetlib.dhp</groupId>
        <artifactId>dhp</artifactId>
        <version>1.2.4-SNAPSHOT</version>
        <relativePath>../pom.xml</relativePath>
    </parent>

    <artifactId>dhp-workflows</artifactId>
    <packaging>pom</packaging>

    <description>This module is the container for the oozie workflow definitions in dnet-hadoop project</description>

    <modules>
        <module>dhp-workflow-profiles</module>
        <module>dhp-aggregation</module>
        <module>dhp-distcp</module>
        <module>dhp-actionmanager</module>
        <module>dhp-graph-mapper</module>
        <module>dhp-dedup-openaire</module>
        <module>dhp-enrichment</module>
        <module>dhp-graph-provision</module>
        <module>dhp-dedup-scholexplorer</module>
        <module>dhp-graph-provision-scholexplorer</module>
        <module>dhp-blacklist</module>
        <module>dhp-stats-update</module>
        <module>dhp-broker-events</module>
        <module>dhp-doiboost</module>
    </modules>

    <pluginRepositories>
        <pluginRepository>
            <id>iis-releases</id>
            <name>iis releases plugin repository</name>
            <url>http://maven.ceon.pl/artifactory/iis-releases</url>
            <layout>default</layout>
        </pluginRepository>
    </pluginRepositories>
    <properties>
        <maven.build.timestamp.format>yyyy-MM-dd_HH_mm</maven.build.timestamp.format>
        <!-- default Oozie installer properties requred to be defined at pom.xml level -->
        <!-- other project properties are defined in project-default.properties -->
        <oozie.package.file.name>oozie-package</oozie.package.file.name>
        <!-- notice: sandboxName is generated based on workflow.source.dir property -->
        <workflow.source.dir>src/test/resources/define/path/pointing/to/directory/holding/oozie_app</workflow.source.dir>
        <oozieAppDir>oozie_app</oozieAppDir>
        <queueName>default</queueName>
        <importerQueueName>default</importerQueueName>
        <oozieLauncherQueueName>default</oozieLauncherQueueName>
        <primed.dir>primed</primed.dir>

        <oozie.package.dependencies.include.scope>runtime</oozie.package.dependencies.include.scope>
        <oozie.package.dependencies.exclude.scope />
        <oozie.package.skip.test.jar>true</oozie.package.skip.test.jar>

        <dhpConnectionProperties>${user.home}/.dhp/application.properties</dhpConnectionProperties>

        <output.dir.name>${maven.build.timestamp}</output.dir.name>

        <projectVersion>${project.version}</projectVersion>
        <oozie.use.system.libpath>true</oozie.use.system.libpath>
    </properties>
    <dependencies>
        <dependency>
            <groupId>eu.dnetlib.dhp</groupId>
            <artifactId>dhp-build-assembly-resources</artifactId>
            <version>${project.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.oozie</groupId>
            <artifactId>oozie-client</artifactId>
        </dependency>
        <dependency>
            <groupId>net.schmizz</groupId>
            <artifactId>sshj</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>
    <profiles>
        <profile>
            <!-- This profile sets properties that are required for test oozie workflows To be used only with 'oozie-package' profile -->
            <id>attach-test-resources</id>
            <properties>
                <!--overriding default scope (set to 'runtime') with the 'test' value. Test resources attached to oozie package requires all test dependencies. -->
                <oozie.package.dependencies.include.scope />
                <oozie.package.dependencies.exclude.scope>provided</oozie.package.dependencies.exclude.scope>
                <!-- Do not skip creation of test jar for priming (in oozie-package profile) -->
                <oozie.package.skip.test.jar>false</oozie.package.skip.test.jar>
            </properties>
        </profile>
        <profile>
            <id>oozie-package</id>
            <build>
                <plugins>
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-enforcer-plugin</artifactId>
                        <version>1.4.1</version>
                        <executions>
                            <execution>
                                <id>enforce-connection-properties-file-existence</id>
                                <phase>initialize</phase>
                                <goals>
                                    <goal>enforce</goal>
                                </goals>
                                <configuration>
                                    <rules>
                                        <requireFilesExist>
                                            <files>
                                                <file>${dhpConnectionProperties}</file>
                                            </files>
                                            <message>
                                                The file with connection properties could not be found. Please, create the ${dhpConnectionProperties} file or set the location to another already created file by using
                                                -DdhpConnectionProperties property.
                                            </message>
                                        </requireFilesExist>
                                    </rules>
                                    <fail>true</fail>
                                </configuration>
                            </execution>
                        </executions>
                    </plugin>
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-dependency-plugin</artifactId>
                        <executions>
                            <execution>
                                <id>copy dependencies</id>
                                <phase>prepare-package</phase>
                                <goals>
                                    <goal>copy-dependencies</goal>
                                </goals>
                                <configuration>
                                    <includeScope>${oozie.package.dependencies.include.scope}</includeScope>
                                    <excludeScope>${oozie.package.dependencies.exclude.scope}</excludeScope>
                                    <silent>true</silent>
                                </configuration>
                            </execution>
                        </executions>
                    </plugin>
                    <!-- Plugin originally defined in attach-test-resources It was moved here to ensure that it will execute before priming -->
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-jar-plugin</artifactId>
                        <executions>
                            <execution>
                                <id>attach-test-resources-package</id>
                                <phase>prepare-package</phase>
                                <goals>
                                    <goal>test-jar</goal>
                                </goals>
                                <configuration>
                                    <skip>${oozie.package.skip.test.jar}</skip>
                                </configuration>
                            </execution>
                        </executions>
                    </plugin>

                    <plugin>
                        <groupId>eu.dnetlib.primer</groupId>
                        <artifactId>primer-maven-plugin</artifactId>
                        <version>1.2.0</version>
                        <executions>
                            <execution>
                                <id>priming</id>
                                <phase>prepare-package</phase>
                                <goals>
                                    <goal>prime</goal>
                                </goals>
                                <configuration>
                                    <classProviderFiles>
                                        <classProviderFile>${project.build.directory}/dependency/*.jar</classProviderFile>
                                        <classProviderFile>${project.build.directory}/*-tests.jar</classProviderFile>
                                        <classProviderFile>${project.build.directory}/classes</classProviderFile>
                                    </classProviderFiles>
                                    <coansysPackageDir>${project.build.directory}/dependency</coansysPackageDir>
                                    <destination>${project.build.directory}/${primed.dir}</destination>
                                    <classpath>${workflow.source.dir}</classpath>
                                </configuration>
                            </execution>
                        </executions>
                    </plugin>
                    <!-- reading job.properties to use them in .sh scripts -->
                    <plugin>
                        <groupId>org.kuali.maven.plugins</groupId>
                        <artifactId>properties-maven-plugin</artifactId>
                        <version>${properties.maven.plugin.version}</version>
                        <dependencies>
                            <dependency>
                                <groupId>eu.dnetlib.dhp</groupId>
                                <artifactId>dhp-build-assembly-resources</artifactId>
                                <version>${project.version}</version>
                                <!-- contains project-default.properties -->
                            </dependency>
                        </dependencies>
                        <executions>
                            <execution>
                                <id>reading-dhp-properties</id>
                                <phase>initialize</phase>
                                <goals>
                                    <goal>read-project-properties</goal>
                                </goals>
                                <configuration>
                                    <locations>
                                        <location>${dhpConnectionProperties}</location>
                                    </locations>
                                    <quiet>false</quiet>
                                </configuration>
                            </execution>
                            <execution>
                                <id>read-default-properties</id>
                                <phase>prepare-package</phase>
                                <goals>
                                    <goal>read-project-properties</goal>
                                </goals>
                                <configuration>
                                    <locations>
                                        <location>classpath:project-default.properties</location>
                                    </locations>
                                    <quiet>true</quiet>
                                </configuration>
                            </execution>
                            <execution>
                                <id>read-job-properties</id>
                                <phase>prepare-package</phase>
                                <goals>
                                    <goal>read-project-properties</goal>
                                </goals>
                                <configuration>
                                    <locations>
                                        <param>${project.build.directory}/${primed.dir}/job.properties</param>
                                        <param>job-override.properties</param>
                                    </locations>
                                    <quiet>true</quiet>
                                </configuration>
                            </execution>
                        </executions>
                    </plugin>
                    <plugin>
                        <groupId>eu.dnetlib.dhp</groupId>
                        <artifactId>dhp-build-properties-maven-plugin</artifactId>
                        <version>${project.version}</version>
                        <executions>
                            <execution>
                                <phase>validate</phase>
                                <goals>
                                    <goal>generate-properties</goal>
                                    <!-- generates sandboxName based on workflow.source.dir when not specified as commandline parameter -->
                                </goals>
                                <configuration>
                                </configuration>
                            </execution>
                            <execution>
                                <id>write-job-properties</id>
                                <phase>prepare-package</phase>
                                <goals>
                                    <goal>write-project-properties</goal>
                                </goals>
                                <configuration>
                                    <outputFile>target/${oozie.package.file.name}/job.properties</outputFile>
                                    <!-- notice: dots are not allowed for job.properties! -->
                                    <include>nameNode,jobTracker,queueName,importerQueueName,oozieLauncherQueueName,
                                        workingDir,oozieTopWfApplicationPath,oozieServiceLoc,
                                        sparkDriverMemory,sparkExecutorMemory,sparkExecutorCores,
                                        oozie.wf.application.path,projectVersion,oozie.use.system.libpath,
                                        oozieActionShareLibForSpark1,spark1YarnHistoryServerAddress,spark1EventLogDir,
                                        oozieActionShareLibForSpark2,spark2YarnHistoryServerAddress,spark2EventLogDir,
                                        sparkSqlWarehouseDir
                                    </include>
                                    <includeSystemProperties>true</includeSystemProperties>
                                    <includePropertyKeysFromFiles>
                                        <!-- <param>${workflow.source.dir}/job.properties</param> -->
                                        <param>${project.build.directory}/${primed.dir}/job.properties</param>
                                        <param>job-override.properties</param>

                                    </includePropertyKeysFromFiles>
                                </configuration>
                            </execution>
                        </executions>
                    </plugin>
                    <plugin>
                        <groupId>pl.project13.maven</groupId>
                        <artifactId>git-commit-id-plugin</artifactId>
                        <version>2.1.11</version>
                        <executions>
                            <execution>
                                <goals>
                                    <goal>revision</goal>
                                </goals>
                            </execution>
                        </executions>
                        <configuration>
                            <verbose>true</verbose>
                            <dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
                            <generateGitPropertiesFile>true</generateGitPropertiesFile>
                            <generateGitPropertiesFilename>target/${oozie.package.file.name}/${oozieAppDir}/version.properties</generateGitPropertiesFilename>
                        </configuration>
                    </plugin>
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-assembly-plugin</artifactId>
                        <version>3.0.0</version>
                        <dependencies>
                            <dependency>
                                <groupId>eu.dnetlib.dhp</groupId>
                                <artifactId>dhp-build-assembly-resources</artifactId>
                                <version>${project.version}</version>
                            </dependency>
                        </dependencies>
                        <executions>
                            <execution>
                                <id>assembly-oozie-installer</id>
                                <phase>package</phase>
                                <goals>
                                    <goal>single</goal>
                                </goals>
                                <configuration>
                                    <appendAssemblyId>false</appendAssemblyId>
                                    <finalName>${oozie.package.file.name}_shell_scripts</finalName>
                                    <descriptorRefs>
                                        <descriptorRef>oozie-installer</descriptorRef>
                                    </descriptorRefs>
                                </configuration>
                            </execution>
                        </executions>
                    </plugin>


                    <plugin>
                        <!-- this plugin prepares oozie installer package-->

                        <artifactId>maven-antrun-plugin</artifactId>
                        <executions>
                            <!-- extracting shared resources phase -->
                            <execution>
                                <id>installer-copy-custom</id>
                                <phase>process-resources</phase>
                                <goals>
                                    <goal>run</goal>
                                </goals>
                                <configuration>
                                    <tasks>
                                        <property name="assembly-resources.loc" value="${maven.dependency.eu.dnetlib.dhp.dhp-build-assembly-resources.jar.path}" />
                                        <unjar src="${assembly-resources.loc}" dest="${project.build.directory}/assembly-resources" />
                                    </tasks>
                                </configuration>
                            </execution>
                            <!-- packaging phase -->
                            <execution>
                                <phase>package</phase>
                                <configuration>
                                    <tasks>
                                        <!-- copying workflow resources -->
                                        <mkdir dir="target/${oozie.package.file.name}" />
                                        <mkdir dir="target/${oozie.package.file.name}/${oozieAppDir}" />
                                        <copy todir="target/${oozie.package.file.name}/${oozieAppDir}">
                                            <!-- <fileset dir="${workflow.source.dir}/${oozieAppDir}" /> replacing with primed dir location -->
                                            <fileset dir="target/${primed.dir}/${oozieAppDir}" />
                                        </copy>
                                        <!-- copying all jars to oozie lib directory -->
                                        <mkdir dir="target/${oozie.package.file.name}/${oozieAppDir}/lib" />
                                        <copy todir="target/${oozie.package.file.name}/${oozieAppDir}/lib">
                                            <fileset dir="${project.build.directory}/dependency" />
                                        </copy>
                                        <!-- copying current module lib -->
                                        <copy todir="target/${oozie.package.file.name}/${oozieAppDir}/lib">
                                            <fileset dir="${project.build.directory}">
                                                <include name="*.jar" />
                                            </fileset>
                                        </copy>


                                        <fixcrlf srcdir="target/${oozie.package.file.name}/${oozieAppDir}/" encoding="UTF-8" outputencoding="UTF-8" includes="**/*.sh,**/*.json,**/*.py,**/*.sql" eol="lf" />


                                        <!-- creating tar.gz package -->
                                        <tar destfile="target/${oozie.package.file.name}.tar.gz" compression="gzip" longfile="gnu">
                                            <tarfileset dir="target/${oozie.package.file.name}" />
                                            <tarfileset dir="target/${oozie.package.file.name}_shell_scripts" filemode="0755">
                                                <include name="**/*.sh" />
                                            </tarfileset>
                                            <tarfileset dir="target/${oozie.package.file.name}_shell_scripts" filemode="0644">
                                                <exclude name="**/*.sh" />
                                            </tarfileset>
                                        </tar>
                                        <!-- cleanup -->
                                        <delete dir="target/${oozie.package.file.name}" />
                                        <delete dir="target/${oozie.package.file.name}_shell_scripts" />
                                    </tasks>
                                </configuration>
                                <goals>
                                    <goal>run</goal>
                                </goals>
                            </execution>
                        </executions>
                    </plugin>
                </plugins>
            </build>
        </profile>
        <profile>
            <id>deploy</id>
            <build>
                <plugins>
                    <plugin>
                        <groupId>org.codehaus.mojo</groupId>
                        <artifactId>exec-maven-plugin</artifactId>
                        <version>1.5.0</version>
                        <executions>
                            <execution>
                                <id>create-target-dir</id>
                                <phase>package</phase>
                                <goals>
                                    <goal>exec</goal>
                                </goals>
                                <configuration>
                                    <executable>ssh</executable>
                                    <arguments>
                                        <argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
                                        <argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
                                        <argument>-o StrictHostKeyChecking=no</argument>
                                        <argument>rm -rf ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; mkdir -p ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/</argument>
                                    </arguments>
                                </configuration>
                            </execution>
                            <execution>
                                <id>upload-oozie-package</id>
                                <phase>package</phase>
                                <goals>
                                    <goal>exec</goal>
                                </goals>
                                <configuration>
                                    <executable>scp</executable>
                                    <arguments>
                                        <argument>-P ${dhp.hadoop.frontend.port.ssh}</argument>
                                        <argument>-o StrictHostKeyChecking=no</argument>
                                        <argument>target/${oozie.package.file.name}.tar.gz</argument>
                                        <argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}:${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/${oozie.package.file.name}.tar.gz</argument>
                                    </arguments>
                                </configuration>
                            </execution>
                            <execution>
                                <id>extract-and-upload-to-hdfs</id>
                                <phase>package</phase>
                                <goals>
                                    <goal>exec</goal>
                                </goals>
                                <configuration>
                                    <executable>ssh</executable>
                                    <!-- <outputFile>target/redirected_upload.log</outputFile> -->
                                    <arguments>
                                        <argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
                                        <argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
                                        <argument>-o StrictHostKeyChecking=no</argument>
                                        <argument>cd ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; </argument>
                                        <argument>tar -zxf oozie-package.tar.gz; </argument>
                                        <argument>rm ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/oozie-package.tar.gz; </argument>
                                        <argument>./upload_workflow.sh</argument>
                                    </arguments>
                                </configuration>
                            </execution>
                        </executions>
                    </plugin>
                </plugins>
            </build>
        </profile>
        <profile>
            <id>run</id>
            <build>
                <plugins>
                    <plugin>
                        <groupId>org.codehaus.mojo</groupId>
                        <artifactId>exec-maven-plugin</artifactId>
                        <version>1.5.0</version>
                        <executions>
                            <execution>
                                <id>run-job</id>
                                <phase>package</phase>
                                <goals>
                                    <goal>exec</goal>
                                </goals>
                                <configuration>
                                    <executable>ssh</executable>
                                    <!-- this file will be used by test verification profile reading job identifier -->
                                    <outputFile>${oozie.execution.log.file.location}</outputFile>
                                    <arguments>
                                        <argument>${dhp.hadoop.frontend.user.name}@${dhp.hadoop.frontend.host.name}</argument>
                                        <argument>-p ${dhp.hadoop.frontend.port.ssh}</argument>
                                        <argument>-o StrictHostKeyChecking=no</argument>
                                        <argument>cd ${dhp.hadoop.frontend.temp.dir}/oozie-packages/${sandboxName}/${output.dir.name}/; </argument>
                                        <argument>./run_workflow.sh</argument>
                                    </arguments>
                                </configuration>
                            </execution>
                            <execution>
                                <id>show-run-log-on-stdout</id>
                                <phase>package</phase>
                                <goals>
                                    <goal>exec</goal>
                                </goals>
                                <configuration>
                                    <executable>cat</executable>
                                    <arguments>
                                        <argument>${oozie.execution.log.file.location}</argument>
                                    </arguments>
                                </configuration>
                            </execution>
                        </executions>
                    </plugin>
                </plugins>
            </build>
        </profile>
        <profile>
            <!-- this profile is handling unit and integration test definitions of all child modules -->
            <id>child-tests</id>
            <build>
                <plugins>
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-jar-plugin</artifactId>
                        <executions>
                            <execution>
                                <id>integration-test-package</id>
                                <phase>integration-test</phase>
                                <goals>
                                    <goal>test-jar</goal>
                                </goals>
                            </execution>
                        </executions>
                    </plugin>
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-surefire-plugin</artifactId>
                        <!--
                        <configuration>
                            <excludedGroups>eu.dnetlib.iis.common.IntegrationTest</excludedGroups>
                        </configuration>
                        -->
                    </plugin>
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-failsafe-plugin</artifactId>
                        <version>${maven.failsave.plugin.version}</version>
                        <executions>
                            <execution>
                                <id>default-integration-test</id>
                                <configuration>
                                    <argLine>-Xmx1024m</argLine>
                                    <systemPropertiesVariables>
                                        <!-- if dhpConnectionProperties is not defined, then ${user.home}/.dhp/integration-test.properties will be used -->
                                        <dhpConnectionProperties>${dhpConnectionProperties}</dhpConnectionProperties>
                                        <output.dir.name>${output.dir.name}</output.dir.name>
                                    </systemPropertiesVariables>
                                </configuration>
                            </execution>
                        </executions>
                    </plugin>
                </plugins>
            </build>
            <activation>
                <file>
                    <missing>src/main/resources/parent.marker</missing>
                </file>
            </activation>
        </profile>
    </profiles>

</project>
