From 2e2d4c4c68d20a0035eb11accc16ef5ff615e275 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 30 Mar 2020 13:11:49 +0200 Subject: [PATCH 1/6] adjusted path to template resource --- .../dhp/oa/provision/utils/TemplateResources.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateResources.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateResources.java index a9086f7bc..b22e083ce 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateResources.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateResources.java @@ -7,17 +7,17 @@ import java.nio.charset.StandardCharsets; public class TemplateResources { - private String record = read("eu/dnetlib/dhp/graph/template/record.st"); + private String record = read("eu/dnetlib/dhp/oa/provision/template/record.st"); - private String instance = read("eu/dnetlib/dhp/graph/template/instance.st"); + private String instance = read("eu/dnetlib/dhp/oa/provision/template/instance.st"); - private String rel = read("eu/dnetlib/dhp/graph/template/rel.st"); + private String rel = read("eu/dnetlib/dhp/oa/provision/template/rel.st"); - private String webresource = read("eu/dnetlib/dhp/graph/template/webresource.st"); + private String webresource = read("eu/dnetlib/dhp/oa/provision/template/webresource.st"); - private String child = read("eu/dnetlib/dhp/graph/template/child.st"); + private String child = read("eu/dnetlib/dhp/oa/provision/template/child.st"); - private String entity = read("eu/dnetlib/dhp/graph/template/entity.st"); + private String entity = read("eu/dnetlib/dhp/oa/provision/template/entity.st"); private static String read(final String classpathResource) throws IOException { return Resources.toString(Resources.getResource(classpathResource), StandardCharsets.UTF_8); From f3f9affd4910b3bbc07fdec0aacc02c4d32d6550 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 30 Mar 2020 13:12:11 +0200 Subject: [PATCH 2/6] allow dynamic executors to build XML records --- .../resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index 1d99831e4..2eeab2906 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -67,7 +67,6 @@ --executor-cores ${sparkExecutorCoresForJoining} --executor-memory ${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} - --conf spark.dynamicAllocation.maxExecutors=${sparkExecutorCoresForJoining} --conf spark.extraListeners="com.cloudera.spark.lineage.NavigatorAppListener" --conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener" --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} From 0fbec69b82f2f730b1f1470b2f04a252f46255e5 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 30 Mar 2020 19:48:41 +0200 Subject: [PATCH 3/6] use oozie prepare statement to cleanup working directories --- .../dnetlib/dhp/oa/provision/SparkXmlRecordBuilderJob.java | 5 ----- .../eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SparkXmlRecordBuilderJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SparkXmlRecordBuilderJob.java index a84cda53a..0a898c0fc 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SparkXmlRecordBuilderJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SparkXmlRecordBuilderJob.java @@ -29,14 +29,9 @@ public class SparkXmlRecordBuilderJob { final String otherDsTypeId = parser.get("otherDsTypeId"); final FileSystem fs = FileSystem.get(spark.sparkContext().hadoopConfiguration()); - if (fs.exists(new Path(outputPath))) { - fs.delete(new Path(outputPath), true); - fs.mkdirs(new Path(outputPath)); - } new GraphJoiner(spark, ContextMapper.fromIS(isLookupUrl), otherDsTypeId, inputPath, outputPath) .adjacencyLists(); - //.asXML(); } } diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index 2eeab2906..a28174cce 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -58,6 +58,10 @@ + + + + yarn cluster build_adjacency_lists From ef429010ee5d177d074ba2fd0b9760e0f1adf718 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 30 Mar 2020 20:00:58 +0200 Subject: [PATCH 4/6] removed log file and job-override.properties --- dhp-workflows/dhp-graph-mapper/derby.log | 13 ------------- .../dhp-graph-provision/job-override.properties | 14 -------------- 2 files changed, 27 deletions(-) delete mode 100644 dhp-workflows/dhp-graph-mapper/derby.log delete mode 100644 dhp-workflows/dhp-graph-provision/job-override.properties diff --git a/dhp-workflows/dhp-graph-mapper/derby.log b/dhp-workflows/dhp-graph-mapper/derby.log deleted file mode 100644 index 0c6791d96..000000000 --- a/dhp-workflows/dhp-graph-mapper/derby.log +++ /dev/null @@ -1,13 +0,0 @@ ----------------------------------------------------------------- -Thu Mar 26 19:43:00 CET 2020: -Booting Derby version The Apache Software Foundation - Apache Derby - 10.12.1.1 - (1704137): instance a816c00e-0171-1827-9724-000012c70f40 -on database directory /private/var/folders/xn/nr5vdk8n1572rvrnx5890_d80000gn/T/junit3871072562876431144/junit_metastore_db with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@4e6b5ed4 -Loaded from file:/Users/claudio/.m2/repository/org/apache/derby/derby/10.12.1.1/derby-10.12.1.1.jar -java.vendor=Oracle Corporation -java.runtime.version=1.8.0_181-b13 -user.dir=/Users/claudio/workspace/git/dnet-hadoop/dhp-workflows/dhp-graph-mapper -os.name=Mac OS X -os.arch=x86_64 -os.version=10.15.3 -derby.system.home=null -Database Class Loader started - derby.database.classpath='' diff --git a/dhp-workflows/dhp-graph-provision/job-override.properties b/dhp-workflows/dhp-graph-provision/job-override.properties deleted file mode 100644 index 8230dfc18..000000000 --- a/dhp-workflows/dhp-graph-provision/job-override.properties +++ /dev/null @@ -1,14 +0,0 @@ -sparkExecutorCoresForJoining=1 -sparkDriverMemoryForJoining=10G -sparkExecutorMemoryForJoining=15G -sparkExecutorCoresForIndexing=64 -sparkDriverMemoryForIndexing=3G -sparkExecutorMemoryForIndexing=2G -#isLookupUrl=http://services.openaire.eu:8280/is/services/isLookUp -isLookupUrl=http://beta.services.openaire.eu:8280/is/services/isLookUp?wsdl -sourcePath=/tmp/db_openaireplus_services.export_dhp.2020.02.03 -outputPath=/tmp/openaire_provision -format=TMF -batchSize=2000 -reuseRecords=false -otherDsTypeId=scholarcomminfra, infospace, pubsrepository::mock, entityregistry, entityregistry::projects, entityregistry::repositories, websource \ No newline at end of file From 76d9315129ba009ca020a79e34ff6ec07f0e2560 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 30 Mar 2020 20:05:56 +0200 Subject: [PATCH 5/6] [maven-release-plugin] prepare release dhp-1.1.6 --- dhp-build/dhp-build-assembly-resources/pom.xml | 2 +- dhp-build/dhp-build-properties-maven-plugin/pom.xml | 4 ++-- dhp-build/pom.xml | 2 +- dhp-common/pom.xml | 2 +- dhp-schemas/pom.xml | 2 +- dhp-workflows/dhp-aggregation/pom.xml | 2 +- dhp-workflows/dhp-dedup-openaire/pom.xml | 2 +- dhp-workflows/dhp-dedup-scholexplorer/pom.xml | 5 ++--- dhp-workflows/dhp-distcp/pom.xml | 2 +- dhp-workflows/dhp-graph-mapper/pom.xml | 5 ++--- dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml | 5 ++--- dhp-workflows/dhp-graph-provision/pom.xml | 2 +- dhp-workflows/pom.xml | 2 +- pom.xml | 8 +++----- 14 files changed, 20 insertions(+), 25 deletions(-) diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml index 0c4637def..99e4593fd 100644 --- a/dhp-build/dhp-build-assembly-resources/pom.xml +++ b/dhp-build/dhp-build-assembly-resources/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.1.6-SNAPSHOT + 1.1.6 dhp-build-assembly-resources diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml index 308d78715..45b3c22e6 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/pom.xml +++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.1.6-SNAPSHOT + 1.1.6 dhp-build-properties-maven-plugin @@ -102,7 +102,7 @@ - + diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml index e471af76d..f9545eab4 100644 --- a/dhp-build/pom.xml +++ b/dhp-build/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp - 1.1.6-SNAPSHOT + 1.1.6 dhp-build pom diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index f6283d450..b92159b39 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.1.6-SNAPSHOT + 1.1.6 ../ diff --git a/dhp-schemas/pom.xml b/dhp-schemas/pom.xml index a85c0dd23..e124829e3 100644 --- a/dhp-schemas/pom.xml +++ b/dhp-schemas/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.1.6-SNAPSHOT + 1.1.6 ../ diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index 95e957847..c70ca3903 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.1.6-SNAPSHOT + 1.1.6 dhp-aggregation diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index a4793da89..ae6bbece5 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.1.6-SNAPSHOT + 1.1.6 4.0.0 dhp-dedup-openaire diff --git a/dhp-workflows/dhp-dedup-scholexplorer/pom.xml b/dhp-workflows/dhp-dedup-scholexplorer/pom.xml index 387952e33..bb059e169 100644 --- a/dhp-workflows/dhp-dedup-scholexplorer/pom.xml +++ b/dhp-workflows/dhp-dedup-scholexplorer/pom.xml @@ -1,10 +1,9 @@ - + dhp-workflows eu.dnetlib.dhp - 1.1.6-SNAPSHOT + 1.1.6 4.0.0 diff --git a/dhp-workflows/dhp-distcp/pom.xml b/dhp-workflows/dhp-distcp/pom.xml index c3f09b42c..de3c5babc 100644 --- a/dhp-workflows/dhp-distcp/pom.xml +++ b/dhp-workflows/dhp-distcp/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.1.6-SNAPSHOT + 1.1.6 4.0.0 diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index 03b6d0052..63e2c94fe 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -1,10 +1,9 @@ - + dhp-workflows eu.dnetlib.dhp - 1.1.6-SNAPSHOT + 1.1.6 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml b/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml index bb41858a6..7dd57ebf6 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml @@ -1,10 +1,9 @@ - + dhp-workflows eu.dnetlib.dhp - 1.1.6-SNAPSHOT + 1.1.6 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index ac4e01d21..fb4bac3c7 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.1.6-SNAPSHOT + 1.1.6 4.0.0 diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index 433cf1fa9..45f705cf9 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp - 1.1.6-SNAPSHOT + 1.1.6 ../ diff --git a/pom.xml b/pom.xml index 99e40b9b5..fb739f938 100644 --- a/pom.xml +++ b/pom.xml @@ -1,11 +1,9 @@ - + 4.0.0 eu.dnetlib.dhp dhp - 1.1.6-SNAPSHOT + 1.1.6 pom http://www.d-net.research-infrastructures.eu @@ -40,7 +38,7 @@ scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git https://code-repo.d4science.org/D-Net/dnet-hadoop/ - HEAD + dhp-1.1.6 From 377e1ba84035f0992e297666063db57a19c9d250 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 30 Mar 2020 20:06:00 +0200 Subject: [PATCH 6/6] [maven-release-plugin] prepare for next development iteration --- dhp-build/dhp-build-assembly-resources/pom.xml | 2 +- dhp-build/dhp-build-properties-maven-plugin/pom.xml | 2 +- dhp-build/pom.xml | 2 +- dhp-common/pom.xml | 2 +- dhp-schemas/pom.xml | 2 +- dhp-workflows/dhp-aggregation/pom.xml | 2 +- dhp-workflows/dhp-dedup-openaire/pom.xml | 2 +- dhp-workflows/dhp-dedup-scholexplorer/pom.xml | 2 +- dhp-workflows/dhp-distcp/pom.xml | 2 +- dhp-workflows/dhp-graph-mapper/pom.xml | 2 +- dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml | 2 +- dhp-workflows/dhp-graph-provision/pom.xml | 2 +- dhp-workflows/pom.xml | 2 +- pom.xml | 4 ++-- 14 files changed, 15 insertions(+), 15 deletions(-) diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml index 99e4593fd..c837cd538 100644 --- a/dhp-build/dhp-build-assembly-resources/pom.xml +++ b/dhp-build/dhp-build-assembly-resources/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.1.6 + 1.1.7-SNAPSHOT dhp-build-assembly-resources diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml index 45b3c22e6..df5045fcb 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/pom.xml +++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.1.6 + 1.1.7-SNAPSHOT dhp-build-properties-maven-plugin diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml index f9545eab4..3b54b4e61 100644 --- a/dhp-build/pom.xml +++ b/dhp-build/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp - 1.1.6 + 1.1.7-SNAPSHOT dhp-build pom diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index b92159b39..1268afa3a 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.1.6 + 1.1.7-SNAPSHOT ../ diff --git a/dhp-schemas/pom.xml b/dhp-schemas/pom.xml index e124829e3..8deb2eab2 100644 --- a/dhp-schemas/pom.xml +++ b/dhp-schemas/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.1.6 + 1.1.7-SNAPSHOT ../ diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index c70ca3903..8d4d880b3 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.1.6 + 1.1.7-SNAPSHOT dhp-aggregation diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index ae6bbece5..e7f2a926f 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.1.6 + 1.1.7-SNAPSHOT 4.0.0 dhp-dedup-openaire diff --git a/dhp-workflows/dhp-dedup-scholexplorer/pom.xml b/dhp-workflows/dhp-dedup-scholexplorer/pom.xml index bb059e169..e87811cd5 100644 --- a/dhp-workflows/dhp-dedup-scholexplorer/pom.xml +++ b/dhp-workflows/dhp-dedup-scholexplorer/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.1.6 + 1.1.7-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-distcp/pom.xml b/dhp-workflows/dhp-distcp/pom.xml index de3c5babc..d013dd1d9 100644 --- a/dhp-workflows/dhp-distcp/pom.xml +++ b/dhp-workflows/dhp-distcp/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.1.6 + 1.1.7-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index 63e2c94fe..9876edc16 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.1.6 + 1.1.7-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml b/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml index 7dd57ebf6..de38a01b3 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.1.6 + 1.1.7-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index fb4bac3c7..baac163d2 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.1.6 + 1.1.7-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index 45f705cf9..4b5fb4b4f 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp - 1.1.6 + 1.1.7-SNAPSHOT ../ diff --git a/pom.xml b/pom.xml index fb739f938..ae19ddbe5 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.dnetlib.dhp dhp - 1.1.6 + 1.1.7-SNAPSHOT pom http://www.d-net.research-infrastructures.eu @@ -38,7 +38,7 @@ scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git https://code-repo.d4science.org/D-Net/dnet-hadoop/ - dhp-1.1.6 + HEAD