From 5fc09b179cf68d169070b43f47ec8828e3351e49 Mon Sep 17 00:00:00 2001 From: Michele Artini Date: Tue, 11 Feb 2020 12:48:03 +0100 Subject: [PATCH] bug fixing --- dhp-workflows/dhp-aggregation/pom.xml | 6 ++++++ .../dnetlib/dhp/migration/AbstractMigrationExecutor.java | 7 +++++++ .../src/main/java/eu/dnetlib/dhp/migration/DbClient.java | 7 +++++-- .../dhp/migration/migrate_db_entities_parameters.json | 4 ++-- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index d031c0308e..d523945ea8 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -55,6 +55,12 @@ org.mongodb mongo-java-driver + + + org.postgresql + postgresql + 42.2.10 + org.mockito diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMigrationExecutor.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMigrationExecutor.java index 3897905110..bf877dcf35 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMigrationExecutor.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMigrationExecutor.java @@ -10,6 +10,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -42,7 +44,12 @@ public class AbstractMigrationExecutor implements Closeable { private final SequenceFile.Writer writer; + private static final Log log = LogFactory.getLog(AbstractMigrationExecutor.class); + public AbstractMigrationExecutor(final String hdfsPath, final String hdfsNameNode, final String hdfsUser) throws Exception { + + log.info(String.format("Creating SequenceFile Writer, hdfsPath=%s, nameNode=%s, user=%s", hdfsPath, hdfsNameNode, hdfsUser)); + this.writer = SequenceFile.createWriter(getConf(hdfsNameNode, hdfsUser), SequenceFile.Writer.file(new Path(hdfsPath)), SequenceFile.Writer .keyClass(IntWritable.class), SequenceFile.Writer.valueClass(Text.class)); } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/DbClient.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/DbClient.java index e9fee63b9a..246dae4746 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/DbClient.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/DbClient.java @@ -9,6 +9,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.function.Consumer; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -22,7 +23,9 @@ public class DbClient implements Closeable { try { Class.forName("org.postgresql.Driver"); - this.connection = DriverManager.getConnection(address, login, password); + + this.connection = + StringUtils.isNoneBlank(login, password) ? DriverManager.getConnection(address, login, password) : DriverManager.getConnection(address); this.connection.setAutoCommit(false); } catch (final Exception e) { log.error(e.getClass().getName() + ": " + e.getMessage()); @@ -34,7 +37,7 @@ public class DbClient implements Closeable { public void processResults(final String sql, final Consumer consumer) { try (final Statement stmt = connection.createStatement()) { - try (final ResultSet rs = stmt.executeQuery("SELECT * FROM COMPANY;")) { + try (final ResultSet rs = stmt.executeQuery(sql)) { while (rs.next()) { consumer.accept(rs); } diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/migrate_db_entities_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/migrate_db_entities_parameters.json index 861d297bae..5e9f378f55 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/migrate_db_entities_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/migrate_db_entities_parameters.json @@ -27,12 +27,12 @@ "paramName": "dbuser", "paramLongName": "postgresUser", "paramDescription": "postgres user", - "paramRequired": true + "paramRequired": false }, { "paramName": "dbpasswd", "paramLongName": "postgresPassword", "paramDescription": "postgres password", - "paramRequired": true + "paramRequired": false } ] \ No newline at end of file