D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

Changes in maven poms to build and test the project using Spark 3.4.x and scala 2.12

This commit is contained in:
Giambattista Bloisi 2023-08-02 18:05:53 +02:00 committed by Giambattista Bloisi
parent 321922772b
commit 326c9dc08c
6 changed files with 116 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
dhp-common/pom.xml
View File

@ -62,16 +62,17 @@
</build> </build>
<dependencies> <dependencies>
<dependency>
<groupId>edu.cmu</groupId>
<artifactId>secondstring</artifactId>
</dependency>
<dependency> <dependency>
<groupId>eu.dnetlib.dhp</groupId> <groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-pace-core</artifactId> <artifactId>dhp-pace-core</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency> <dependency>
<groupId>com.github.sisyphsu</groupId> <groupId>com.github.sisyphsu</groupId>
<artifactId>dateparser</artifactId> <artifactId>dateparser</artifactId>
@ -118,10 +119,6 @@
<groupId>net.sf.saxon</groupId> <groupId>net.sf.saxon</groupId>
<artifactId>Saxon-HE</artifactId> <artifactId>Saxon-HE</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.apache.cxf</groupId> <groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-transports-http</artifactId> <artifactId>cxf-rt-transports-http</artifactId>
@ -129,6 +126,12 @@
<dependency> <dependency>
<groupId>eu.dnetlib</groupId> <groupId>eu.dnetlib</groupId>
<artifactId>cnr-rmi-api</artifactId> <artifactId>cnr-rmi-api</artifactId>
<exclusions>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>

12
dhp-pace-core/pom.xml
View File

@ -53,14 +53,6 @@
<groupId>edu.cmu</groupId> <groupId>edu.cmu</groupId>
<artifactId>secondstring</artifactId> <artifactId>secondstring</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
@ -85,10 +77,6 @@
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId> <artifactId>jackson-databind</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
</dependency>
<dependency> <dependency>
<groupId>com.jayway.jsonpath</groupId> <groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId> <artifactId>json-path</artifactId>

24
dhp-workflows/dhp-dedup-openaire/pom.xml
View File

@ -54,24 +54,10 @@
<artifactId>dhp-pace-core</artifactId> <artifactId>dhp-pace-core</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-java8-compat_${scala.binary.version}</artifactId>
<version>1.0.2</version>
</dependency>
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-collection-compat_${scala.binary.version}</artifactId>
<version>2.11.0</version>
</dependency>
<dependency> <dependency>
<groupId>org.apache.spark</groupId> <groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId> <artifactId>spark-core_${scala.binary.version}</artifactId>
@ -80,16 +66,10 @@
<groupId>org.apache.spark</groupId> <groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId> <artifactId>spark-sql_${scala.binary.version}</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.spark</groupId> <groupId>org.apache.spark</groupId>
<artifactId>spark-graphx_${scala.binary.version}</artifactId> <artifactId>spark-graphx_${scala.binary.version}</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.arakelian</groupId>
<artifactId>java-jq</artifactId>
</dependency>
<dependency> <dependency>
<groupId>dom4j</groupId> <groupId>dom4j</groupId>
<artifactId>dom4j</artifactId> <artifactId>dom4j</artifactId>
@ -102,10 +82,6 @@
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId> <artifactId>jackson-databind</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId> <artifactId>httpclient</artifactId>

14
dhp-workflows/dhp-graph-provision/pom.xml
View File

@ -59,12 +59,6 @@
<dependency> <dependency>
<groupId>com.jayway.jsonpath</groupId> <groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId> <artifactId>json-path</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>dom4j</groupId> <groupId>dom4j</groupId>
@ -160,6 +154,14 @@
<groupId>org.apache.zookeeper</groupId> <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId> <artifactId>zookeeper</artifactId>
</exclusion> </exclusion>
<exclusion>
<artifactId>ant</artifactId>
<groupId>org.apache.ant</groupId>
</exclusion>
<exclusion>
<artifactId>antlr4-runtime</artifactId>
<groupId>org.antlr</groupId>
</exclusion>
</exclusions> </exclusions>
</dependency> </dependency>
<dependency> <dependency>

3
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateFactory.java
View File

@ -5,14 +5,11 @@ import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.removePrefix;
import static eu.dnetlib.dhp.oa.provision.utils.XmlSerializationUtils.escapeXml; import static eu.dnetlib.dhp.oa.provision.utils.XmlSerializationUtils.escapeXml;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import javax.swing.text.html.Option;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.stringtemplate.v4.ST; import org.stringtemplate.v4.ST;

145
pom.xml
View File

@ -204,10 +204,17 @@
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${org.slf4j.version}</version>
<scope>provided</scope>
</dependency>
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId> <artifactId>jcl-over-slf4j</artifactId>
<version>1.7.25</version> <version>${org.slf4j.version}</version>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
@ -217,22 +224,29 @@
<version>${dhp.commons.lang.version}</version> <version>${dhp.commons.lang.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-beanutils</artifactId>
<version>${commons-beanutils.version}</version>
</dependency>
<dependency> <dependency>
<groupId>commons-validator</groupId> <groupId>commons-validator</groupId>
<artifactId>commons-validator</artifactId> <artifactId>commons-validator</artifactId>
<version>1.7</version> <version>${commons-validator.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.github.sisyphsu</groupId> <groupId>com.github.sisyphsu</groupId>
<artifactId>dateparser</artifactId> <artifactId>dateparser</artifactId>
<version>1.0.7</version> <version>${dateparser.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>me.xuender</groupId> <groupId>me.xuender</groupId>
<artifactId>unidecode</artifactId> <artifactId>unidecode</artifactId>
<version>0.0.7</version> <version>${unidecode.version}</version>
</dependency> </dependency>
<dependency> <dependency>
@ -245,13 +259,13 @@
<dependency> <dependency>
<groupId>commons-codec</groupId> <groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId> <artifactId>commons-codec</artifactId>
<version>1.9</version> <version>${commons-codec.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>commons-io</groupId> <groupId>commons-io</groupId>
<artifactId>commons-io</artifactId> <artifactId>commons-io</artifactId>
<version>2.4</version> <version>${commons-io.version}</version>
</dependency> </dependency>
<dependency> <dependency>
@ -415,6 +429,7 @@
<artifactId>cxf-rt-transports-http</artifactId> <artifactId>cxf-rt-transports-http</artifactId>
<version>3.1.5</version> <version>3.1.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>javax.persistence</groupId> <groupId>javax.persistence</groupId>
<artifactId>javax.persistence-api</artifactId> <artifactId>javax.persistence-api</artifactId>
@ -504,16 +519,11 @@
<artifactId>commons-compress</artifactId> <artifactId>commons-compress</artifactId>
<version>${common.compress.version}</version> <version>${common.compress.version}</version>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-csv -->
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId> <artifactId>commons-csv</artifactId>
<version>${common.csv.version}</version> <version>${common.csv.version}</version>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
<dependency> <dependency>
<groupId>org.apache.poi</groupId> <groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId> <artifactId>poi-ooxml</artifactId>
@ -568,14 +578,12 @@
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId> <artifactId>commons-math3</artifactId>
<version>3.6.1</version> <version>3.6.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.code.gson</groupId> <groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId> <artifactId>gson</artifactId>
@ -596,7 +604,7 @@
<dependency> <dependency>
<groupId>org.reflections</groupId> <groupId>org.reflections</groupId>
<artifactId>reflections</artifactId> <artifactId>reflections</artifactId>
<version>0.9.10</version> <version>${reflections.version}</version>
</dependency> </dependency>
<dependency> <dependency>
@ -610,6 +618,12 @@
<artifactId>icu4j</artifactId> <artifactId>icu4j</artifactId>
<version>70.1</version> <version>70.1</version>
</dependency> </dependency>
<dependency>
<groupId>org.javassist</groupId>
<artifactId>javassist</artifactId>
<version>${javassist.version}</version>
</dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
@ -866,46 +880,62 @@
<dhp.site.stage.path>sftp://dnet-hadoop@static-web.d4science.org/dnet-hadoop</dhp.site.stage.path> <dhp.site.stage.path>sftp://dnet-hadoop@static-web.d4science.org/dnet-hadoop</dhp.site.stage.path>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<maven.compiler.plugin.version>3.6.0</maven.compiler.plugin.version>
<maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.target>1.8</maven.compiler.target>
<maven.failsave.plugin.version>2.22.2</maven.failsave.plugin.version>
<properties.maven.plugin.version>2.0.1</properties.maven.plugin.version> <!-- scala version -->
<dhp.cdh.version>cdh5.9.2</dhp.cdh.version>
<dhp.hadoop.version>2.6.0-${dhp.cdh.version}</dhp.hadoop.version>
<dhp.oozie.version>4.1.0-${dhp.cdh.version}</dhp.oozie.version>
<dhp-schemas.artifact>dhp-schemas</dhp-schemas.artifact>
<sparksolr.version>3.6.0</sparksolr.version>
<dhp.spark.version>2.4.0.cloudera2</dhp.spark.version>
<dhp.jackson.version>2.9.6</dhp.jackson.version>
<dhp.commons.lang.version>3.5</dhp.commons.lang.version>
<dhp.site.skip>true</dhp.site.skip>
<dhp.guava.version>11.0.2</dhp.guava.version>
<scala.version>2.11.12</scala.version> <scala.version>2.11.12</scala.version>
<scala.binary.version>2.11</scala.binary.version> <scala.binary.version>2.11</scala.binary.version>
<scala-xml.version>1.3.0</scala-xml.version>
<junit-jupiter.version>5.6.1</junit-jupiter.version> <!-- plugin versions -->
<mockito-core.version>3.3.3</mockito-core.version> <maven.compiler.plugin.version>3.6.0</maven.compiler.plugin.version>
<mongodb.driver.version>3.4.2</mongodb.driver.version> <maven.failsave.plugin.version>2.22.2</maven.failsave.plugin.version>
<vtd.version>[2.12,3.0)</vtd.version> <properties.maven.plugin.version>2.0.1</properties.maven.plugin.version>
<dhp-schemas.version>[4.17.2]</dhp-schemas.version> <net.alchim31.maven.version>4.0.1</net.alchim31.maven.version>
<!-- dependency versions -->
<dhp-schemas.artifact>dhp-schemas</dhp-schemas.artifact>
<apache.poi.version>4.1.2</apache.poi.version>
<cnr-rmi-api.version>[2.6.1]</cnr-rmi-api.version>
<common.compress.version>1.20</common.compress.version>
<common.csv.version>1.8</common.csv.version>
<common.text.version>1.8</common.text.version>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<commons-codec.version>1.9</commons-codec.version>
<commons.collections.version>3.2.1</commons.collections.version>
<commons-io.version>2.4</commons-io.version>
<commons.logging.version>1.1.3</commons.logging.version>
<commons-validator.version>1.7</commons-validator.version>
<dateparser.version>1.0.7</dateparser.version>
<dhp-schemas.version>[3.17.1]</dhp-schemas.version>
<dhp.cdh.version>cdh5.9.2</dhp.cdh.version>
<dhp.commons.lang.version>3.5</dhp.commons.lang.version>
<dhp.guava.version>11.0.2</dhp.guava.version>
<dhp.hadoop.version>2.6.0-${dhp.cdh.version}</dhp.hadoop.version>
<dhp.jackson.version>2.9.6</dhp.jackson.version>
<dhp.oozie.version>4.1.0-${dhp.cdh.version}</dhp.oozie.version>
<dhp.site.skip>true</dhp.site.skip>
<dhp.spark.version>2.4.0.cloudera2</dhp.spark.version>
<dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version> <dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version>
<dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version> <dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version>
<dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version> <dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version>
<cnr-rmi-api.version>[2.6.1]</cnr-rmi-api.version> <google.gson.version>2.2.2</google.gson.version>
<solr.version>7.5.0</solr.version> <javassist.version>3.19.0-GA</javassist.version>
<okhttp.version>4.7.2</okhttp.version>
<common.compress.version>1.20</common.compress.version>
<json4s.version>3.5.3</json4s.version> <json4s.version>3.5.3</json4s.version>
<jsonschemagenerator.version>4.13.0</jsonschemagenerator.version> <jsonschemagenerator.version>4.13.0</jsonschemagenerator.version>
<common.csv.version>1.8</common.csv.version> <junit-jupiter.version>5.6.1</junit-jupiter.version>
<apache.poi.version>4.1.2</apache.poi.version> <mockito-core.version>3.3.3</mockito-core.version>
<common.text.version>1.8</common.text.version> <mongodb.driver.version>3.4.2</mongodb.driver.version>
<okhttp.version>4.7.2</okhttp.version>
<org.apache.httpcomponents.version>4.5.3</org.apache.httpcomponents.version> <org.apache.httpcomponents.version>4.5.3</org.apache.httpcomponents.version>
<net.alchim31.maven.version>4.0.1</net.alchim31.maven.version> <org.slf4j.version>1.7.25</org.slf4j.version>
<google.gson.version>2.2.2</google.gson.version> <reflections.version>0.9.10</reflections.version>
<commons.logging.version>1.1.3</commons.logging.version> <scala-xml.version>1.3.0</scala-xml.version>
<commons.collections.version>3.2.1</commons.collections.version> <solr.version>7.5.0</solr.version>
<sparksolr.version>3.6.0</sparksolr.version>
<unidecode.version>0.0.7</unidecode.version>
<vtd.version>[2.12,3.0)</vtd.version>
</properties> </properties>
<!-- Build with scala 12 and Spark 3.4 --> <!-- Build with scala 12 and Spark 3.4 -->
@ -915,21 +945,40 @@
<properties> <properties>
<scala.binary.version>2.12</scala.binary.version> <scala.binary.version>2.12</scala.binary.version>
<scala.version>2.12.18</scala.version> <scala.version>2.12.18</scala.version>
<!-- scala-xml.version>2.1.0</scala-xml.version --> <scala-xml.version>1.3.0</scala-xml.version>
<!-- plugin versions -->
<net.alchim31.maven.version>4.8.1</net.alchim31.maven.version>
<!-- dependencies -->
<common.compress.version>1.22</common.compress.version>
<common.csv.version>1.8</common.csv.version>
<common.text.version>1.10.0</common.text.version>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<commons-codec.version>1.15</commons-codec.version>
<commons.collections.version>3.2.2</commons.collections.version>
<commons-io.version>2.11.0</commons-io.version>
<commons.logging.version>1.1.3</commons.logging.version>
<commons-validator.version>1.7</commons-validator.version>
<dhp.guava.version>14.0.1</dhp.guava.version>
<solr.version>8.11.0</solr.version>
<sparksolr.version>4.0.2</sparksolr.version> <sparksolr.version>4.0.2</sparksolr.version>
<dhp.spark.version>3.4.1</dhp.spark.version> <dhp.spark.version>3.4.1</dhp.spark.version>
<dhp.jackson.version>2.14.2</dhp.jackson.version> <dhp.jackson.version>2.14.2</dhp.jackson.version>
<dhp.commons.lang.version>3.12.0</dhp.commons.lang.version> <dhp.commons.lang.version>3.12.0</dhp.commons.lang.version>
<json4s.version>3.7.0-M11</json4s.version> <json4s.version>3.7.0-M11</json4s.version>
<net.alchim31.maven.version>4.8.1</net.alchim31.maven.version> <javassist.version>3.25.0-GA</javassist.version>
<okhttp.version>4.10.0</okhttp.version>
<org.slf4j.version>2.0.6</org.slf4j.version>
<reflections.version>0.10.2</reflections.version>
<!-- <!--
<dhp-schemas.artifact>dhp-schemas_${scala.binary.version}</dhp-schemas.artifact> <dhp-schemas.artifact>dhp-schemas_${scala.binary.version}</dhp-schemas.artifact>
<dhp-schemas.version>3.17.2-SNAPSHOT</dhp-schemas.version> <dhp-schemas.version>3.17.2-SNAPSHOT</dhp-schemas.version>
--> -->
</properties> </properties>
</profile> </profile>
</profiles> </profiles>
</project> </project>