From 7da24c1dec2e942b85aa821f2140094fd8886dcf Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 28 Jun 2022 13:47:49 +0200 Subject: [PATCH] added more logging --- .../src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java | 5 +++++ .../oa/graph/raw/MigrateHdfsMdstoresApplication.java | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java index 5a59bc0df..e10d0c500 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java @@ -75,9 +75,14 @@ public class DHPUtils { final HttpGet req = new HttpGet(url); + log.info("MDStoreManager request: {}", req); + try (final CloseableHttpClient client = HttpClients.createDefault()) { try (final CloseableHttpResponse response = client.execute(req)) { final String json = IOUtils.toString(response.getEntity().getContent()); + + log.info("MDStoreManager response: {}", json); + final MDStoreWithInfo[] mdstores = objectMapper.readValue(json, MDStoreWithInfo[].class); return Arrays .stream(mdstores) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateHdfsMdstoresApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateHdfsMdstoresApplication.java index 6c72e4dfc..ab6f54b92 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateHdfsMdstoresApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateHdfsMdstoresApplication.java @@ -65,11 +65,19 @@ public class MigrateHdfsMdstoresApplication extends AbstractMigrationApplication log.info("isSparkSessionManaged: {}", isSparkSessionManaged); final String mdstoreManagerUrl = parser.get("mdstoreManagerUrl"); + log.info("mdstoreManagerUrl: {}", mdstoreManagerUrl); + final String mdFormat = parser.get("mdFormat"); + log.info("mdFormat: {}", mdFormat); + final String mdLayout = parser.get("mdLayout"); + log.info("mdLayout: {}", mdLayout); + final String mdInterpretation = parser.get("mdInterpretation"); + log.info("mdInterpretation: {}", mdInterpretation); final String hdfsPath = parser.get("hdfsPath"); + log.info("hdfsPath: {}", hdfsPath); final Set paths = mdstorePaths(mdstoreManagerUrl, mdFormat, mdLayout, mdInterpretation); @@ -95,6 +103,8 @@ public class MigrateHdfsMdstoresApplication extends AbstractMigrationApplication .filter(p -> HdfsSupport.exists(p, sc.hadoopConfiguration())) .toArray(size -> new String[size]); + log.info("Processing existing paths {}", Arrays.asList(validPaths)); + if (validPaths.length > 0) { spark .read()