From 80cb52593f80c1287498747b69286eaf730db943 Mon Sep 17 00:00:00 2001 From: Michele Artini Date: Thu, 13 Feb 2020 15:34:13 +0100 Subject: [PATCH] bug fixing --- .../migration/AbstractMigrationExecutor.java | 4 +++ .../dhp/migration/AbstractMongoExecutor.java | 5 +++- .../dnetlib/dhp/migration/MdstoreClient.java | 25 ++++++++++++------- .../dhp/migration/OdfMigrationExecutor.java | 1 + 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMigrationExecutor.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMigrationExecutor.java index 11c1fb6ae..e91a53045 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMigrationExecutor.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMigrationExecutor.java @@ -111,6 +111,10 @@ public class AbstractMigrationExecutor implements Closeable { return Arrays.stream(values).map(v -> field(v, info)).filter(Objects::nonNull).collect(Collectors.toList()); } + public static List> listFields(final DataInfo info, final List values) { + return values.stream().map(v -> field(v, info)).filter(Objects::nonNull).collect(Collectors.toList()); + } + public static Qualifier qualifier(final String classid, final String classname, final String schemeid, final String schemename) { final Qualifier q = new Qualifier(); q.setClassid(classid); diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMongoExecutor.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMongoExecutor.java index 83e05c59f..d1b618c7a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMongoExecutor.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMongoExecutor.java @@ -385,6 +385,8 @@ public abstract class AbstractMongoExecutor extends AbstractMigrationExecutor { protected OAIProvenance prepareOAIprovenance(final Document doc) { final Node n = doc.selectSingleNode("//*[local-name()='provenance']/*[local-name()='originDescription']"); + if (n == null) { return null; } + final String identifier = n.valueOf("./*[local-name()='identifier']"); final String baseURL = n.valueOf("./*[local-name()='baseURL']");; final String metadataNamespace = n.valueOf("./*[local-name()='metadataNamespace']");; @@ -393,6 +395,7 @@ public abstract class AbstractMongoExecutor extends AbstractMigrationExecutor { final String harvestDate = n.valueOf("@harvestDate");; return oaiIProvenance(identifier, baseURL, metadataNamespace, altered, datestamp, harvestDate); + } protected DataInfo prepareDataInfo(final Document doc) { @@ -416,7 +419,7 @@ public abstract class AbstractMongoExecutor extends AbstractMigrationExecutor { } protected List> prepareListFields(final Node node, final String xpath, final DataInfo info) { - return listFields(info, (String[]) prepareListString(node, xpath).toArray()); + return listFields(info, prepareListString(node, xpath)); } protected List prepareListString(final Node node, final String xpath) { diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/MdstoreClient.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/MdstoreClient.java index 971d7f165..87dadfc7a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/MdstoreClient.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/MdstoreClient.java @@ -2,6 +2,7 @@ package eu.dnetlib.dhp.migration; import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.stream.StreamSupport; @@ -35,7 +36,7 @@ public class MdstoreClient implements Closeable { public Map validCollections(final String mdFormat, final String mdLayout, final String mdInterpretation) { final Map transactions = new HashMap<>(); - for (final Document entry : getColl(db, COLL_METADATA_MANAGER).find()) { + for (final Document entry : getColl(db, COLL_METADATA_MANAGER, true).find()) { final String mdId = entry.getString("mdId"); final String currentId = entry.getString("currentId"); if (StringUtils.isNoneBlank(mdId, currentId)) { @@ -44,7 +45,7 @@ public class MdstoreClient implements Closeable { } final Map res = new HashMap<>(); - for (final Document entry : getColl(db, COLL_METADATA).find()) { + for (final Document entry : getColl(db, COLL_METADATA, true).find()) { if (entry.getString("format").equals(mdFormat) && entry.getString("layout").equals(mdLayout) && entry.getString("interpretation").equals(mdInterpretation) && transactions.containsKey(entry.getString("mdId"))) { res.put(entry.getString("mdId"), transactions.get(entry.getString("mdId"))); @@ -63,20 +64,26 @@ public class MdstoreClient implements Closeable { return client.getDatabase(dbName); } - private MongoCollection getColl(final MongoDatabase db, final String collName) { + private MongoCollection getColl(final MongoDatabase db, final String collName, final boolean abortIfMissing) { if (!Iterables.contains(db.listCollectionNames(), collName)) { final String err = String.format(String.format("Missing collection '%s' in database '%s'", collName, db.getName())); log.warn(err); - throw new RuntimeException(err); + if (abortIfMissing) { + throw new RuntimeException(err); + } else { + return null; + } } return db.getCollection(collName); } - public Iterable listRecords(final String coll) { - return () -> StreamSupport.stream(getColl(db, coll).find().spliterator(), false) - .filter(e -> e.containsKey("body")) - .map(e -> e.getString("body")) - .iterator(); + public Iterable listRecords(final String collName) { + final MongoCollection coll = getColl(db, collName, false); + return coll == null ? new ArrayList<>() + : () -> StreamSupport.stream(coll.find().spliterator(), false) + .filter(e -> e.containsKey("body")) + .map(e -> e.getString("body")) + .iterator(); } @Override diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/OdfMigrationExecutor.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/OdfMigrationExecutor.java index b1dbfcdf4..54636b3bf 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/OdfMigrationExecutor.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/OdfMigrationExecutor.java @@ -34,6 +34,7 @@ public class OdfMigrationExecutor extends AbstractMongoExecutor { @Override protected void registerNamespaces(final Map nsContext) { + super.registerNamespaces(nsContext); nsContext.put("dc", "http://datacite.org/schema/kernel-3"); }