From 54ca8ed6c3c9c34bd39e36af02ca492c8fe25b19 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 29 May 2020 18:17:30 +0200 Subject: [PATCH] uniformed param name (isLookupUrl), Vocab model classes defined as Serializable --- .../graph/raw/GenerateEntitiesApplication.java | 8 ++++++-- .../graph/raw/MigrateDbEntitiesApplication.java | 17 +++++++++++++---- .../dhp/oa/graph/raw/common/Vocabulary.java | 3 ++- .../oa/graph/raw/common/VocabularyGroup.java | 3 ++- .../dhp/oa/graph/raw/common/VocabularyTerm.java | 4 +++- .../oa/graph/generate_entities_parameters.json | 4 ++-- .../graph/migrate_db_entities_parameters.json | 4 ++-- .../dhp/oa/graph/raw_all/oozie_app/workflow.xml | 7 ++++--- .../dhp/oa/graph/raw_db/oozie_app/workflow.xml | 4 ++-- 9 files changed, 36 insertions(+), 18 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java index d6ddc1fdd..8e5ba9cd1 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java @@ -63,9 +63,13 @@ public class GenerateEntitiesApplication { log.info("isSparkSessionManaged: {}", isSparkSessionManaged); final String sourcePaths = parser.get("sourcePaths"); - final String targetPath = parser.get("targetPath"); + log.info("sourcePaths: {}", sourcePaths); - final String isLookupUrl = parser.get("islookup"); + final String targetPath = parser.get("targetPath"); + log.info("targetPath: {}", targetPath); + + final String isLookupUrl = parser.get("isLookupUrl"); + log.info("isLookupUrl: {}", isLookupUrl); final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookupUrl); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java index 899ce6b94..d42754fdc 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java @@ -47,8 +47,6 @@ import java.util.function.Function; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.DbClient; @@ -71,10 +69,12 @@ import eu.dnetlib.dhp.schema.oaf.Relation; import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.dhp.schema.oaf.Software; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MigrateDbEntitiesApplication extends AbstractMigrationApplication implements Closeable { - private static final Log log = LogFactory.getLog(MigrateDbEntitiesApplication.class); + private static final Logger log = LoggerFactory.getLogger(MigrateDbEntitiesApplication.class); public static final String SOURCE_TYPE = "source_type"; public static final String TARGET_TYPE = "target_type"; @@ -95,13 +95,22 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i parser.parseArgument(args); final String dbUrl = parser.get("postgresUrl"); + log.info("postgresUrl: {}", dbUrl); + final String dbUser = parser.get("postgresUser"); + log.info("postgresUser: {}", dbUser); + final String dbPassword = parser.get("postgresPassword"); - final String isLookupUrl = parser.get("islookup"); + log.info("postgresPassword: xxx"); + + final String isLookupUrl = parser.get("isLookupUrl"); + log.info("isLookupUrl: {}", isLookupUrl); final String hdfsPath = parser.get("hdfsPath"); + log.info("hdfsPath: {}", hdfsPath); final boolean processClaims = parser.get("action") != null && parser.get("action").equalsIgnoreCase("claims"); + log.info("processClaims: {}", processClaims); try (final MigrateDbEntitiesApplication smdbe = new MigrateDbEntitiesApplication(hdfsPath, dbUrl, dbUser, dbPassword, isLookupUrl)) { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java index 7714f6d90..c955ee640 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java @@ -1,10 +1,11 @@ package eu.dnetlib.dhp.oa.graph.raw.common; +import java.io.Serializable; import java.util.HashMap; import java.util.Map; -public class Vocabulary { +public class Vocabulary implements Serializable { private final String id; private final String name; diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java index 9991fa4cf..43ed7f2d9 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java @@ -2,6 +2,7 @@ package eu.dnetlib.dhp.oa.graph.raw.common; import java.io.IOException; +import java.io.Serializable; import java.util.HashMap; import java.util.Map; @@ -14,7 +15,7 @@ import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; -public class VocabularyGroup { +public class VocabularyGroup implements Serializable { public static VocabularyGroup loadVocsFromIS(final String isLookupUrl) throws IOException, ISLookUpException { final ISLookUpService isLookUpService = ISLookupClientFactory.getLookUpService(isLookupUrl); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyTerm.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyTerm.java index b3c785923..1aa1b8253 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyTerm.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyTerm.java @@ -1,7 +1,9 @@ package eu.dnetlib.dhp.oa.graph.raw.common; -public class VocabularyTerm { +import java.io.Serializable; + +public class VocabularyTerm implements Serializable { private final String id; private final String name; diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/generate_entities_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/generate_entities_parameters.json index 9e3992bcf..8342dde95 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/generate_entities_parameters.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/generate_entities_parameters.json @@ -18,8 +18,8 @@ "paramRequired": true }, { - "paramName": "islookup", - "paramLongName": "islookup", + "paramName": "isu", + "paramLongName": "isLookupUrl", "paramDescription": "the url of the ISLookupService", "paramRequired": true } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json index edecb97b7..4e838561d 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json @@ -30,8 +30,8 @@ "paramRequired": false }, { - "paramName": "islookup", - "paramLongName": "islookup", + "paramName": "isu", + "paramLongName": "isLookupUrl", "paramDescription": "the url of the ISLookupService", "paramRequired": true } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml index 5eee92342..c2bea9f8a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml @@ -123,6 +123,7 @@ --postgresUrl${postgresURL} --postgresUser${postgresUser} --postgresPassword${postgresPassword} + --isLookupUrl${isLookupUrl} --actionclaims @@ -173,7 +174,7 @@ --postgresUrl${postgresURL} --postgresUser${postgresUser} --postgresPassword${postgresPassword} - --islookup${isLookupUrl} + --isLookupUrl${isLookupUrl} @@ -238,7 +239,7 @@ --sourcePaths${contentPath}/db_claims,${contentPath}/oaf_claims,${contentPath}/odf_claims --targetPath${workingDir}/entities_claim - --islookup${isLookupUrl} + --isLookupUrl${isLookupUrl} @@ -285,7 +286,7 @@ --sourcePaths${contentPath}/db_records,${contentPath}/oaf_records,${contentPath}/odf_records --targetPath${workingDir}/entities - --islookup${isLookupUrl} + --isLookupUrl${isLookupUrl} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_db/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_db/oozie_app/workflow.xml index 93018e674..575f9229e 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_db/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_db/oozie_app/workflow.xml @@ -92,7 +92,7 @@ --postgresUrl${postgresURL} --postgresUser${postgresUser} --postgresPassword${postgresPassword} - --islookup${isLookupUrl} + --isLookupUrl${isLookupUrl} @@ -108,7 +108,7 @@ --postgresUrl${postgresURL} --postgresUser${postgresUser} --postgresPassword${postgresPassword} - --islookup${isLookupUrl} + --isLookupUrl${isLookupUrl} --actionclaims