D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

master #2

Merged
sandro.labruzzo merged 16 commits from michele.artini/dnet-hadoop:master into master 2020-02-17 10:43:09 +01:00
Conversation 0 Commits 16 Files Changed 26 +22 -7
2 changed files with 22 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit cdea0dae75 - Show all commits

19
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMongoExecutor.java
View File

@ -11,6 +11,8 @@ import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.dom4j.Document; import org.dom4j.Document;
import org.dom4j.DocumentException; import org.dom4j.DocumentException;
import org.dom4j.DocumentFactory; import org.dom4j.DocumentFactory;
@ -49,6 +51,8 @@ public abstract class AbstractMongoExecutor extends AbstractMigrationExecutor {
protected static final Qualifier SOFTWARE_RESULTTYPE_QUALIFIER = qualifier("software", "software", "dnet:result_typologies", "dnet:result_typologies"); protected static final Qualifier SOFTWARE_RESULTTYPE_QUALIFIER = qualifier("software", "software", "dnet:result_typologies", "dnet:result_typologies");
protected static final Qualifier OTHER_RESULTTYPE_QUALIFIER = qualifier("other", "other", "dnet:result_typologies", "dnet:result_typologies"); protected static final Qualifier OTHER_RESULTTYPE_QUALIFIER = qualifier("other", "other", "dnet:result_typologies", "dnet:result_typologies");
private static final Log log = LogFactory.getLog(AbstractMongoExecutor.class);
public AbstractMongoExecutor(final String hdfsPath, final String hdfsNameNode, final String hdfsUser, final String mongoBaseUrl, public AbstractMongoExecutor(final String hdfsPath, final String hdfsNameNode, final String hdfsUser, final String mongoBaseUrl,
final String mongoDb, final String dbUrl, final String dbUser, final String mongoDb, final String dbUrl, final String dbUser,
final String dbPassword) throws Exception { final String dbPassword) throws Exception {
@ -66,6 +70,9 @@ public abstract class AbstractMongoExecutor extends AbstractMigrationExecutor {
} }
private void loadClassNames(final String dbUrl, final String dbUser, final String dbPassword) throws IOException { private void loadClassNames(final String dbUrl, final String dbUser, final String dbPassword) throws IOException {
log.info("Loading vocabulary terms from db...");
try (DbClient dbClient = new DbClient(dbUrl, dbUser, dbPassword)) { try (DbClient dbClient = new DbClient(dbUrl, dbUser, dbPassword)) {
code2name.clear(); code2name.clear();
dbClient.processResults("select code, name from class", rs -> { dbClient.processResults("select code, name from class", rs -> {
@ -77,12 +84,19 @@ public abstract class AbstractMongoExecutor extends AbstractMigrationExecutor {
}); });
} }
log.info("Found " + code2name.size() + " terms.");
} }
public void processMdRecords(final String mdFormat, final String mdLayout, final String mdInterpretation) throws DocumentException { public void processMdRecords(final String mdFormat, final String mdLayout, final String mdInterpretation) throws DocumentException {
for (final Entry<String, String> entry : mdstoreClient.validCollections(mdFormat, mdLayout, mdInterpretation).entrySet()) { log.info(String.format("Searching mdstores (format: %s, layout: %s, interpretation: %s)", mdFormat, mdLayout, mdInterpretation));
// final String mdId = entry.getKey();
final Map<String, String> colls = mdstoreClient.validCollections(mdFormat, mdLayout, mdInterpretation);
log.info("Found " + colls.size() + " mdstores");
for (final Entry<String, String> entry : colls.entrySet()) {
log.info("Processing mdstore " + entry.getKey() + " (collection: " + entry.getValue() + ")");
final String currentColl = entry.getValue(); final String currentColl = entry.getValue();
for (final String xml : mdstoreClient.listRecords(currentColl)) { for (final String xml : mdstoreClient.listRecords(currentColl)) {
@ -101,6 +115,7 @@ public abstract class AbstractMongoExecutor extends AbstractMigrationExecutor {
} }
} }
} }
log.info("All Done.");
} }
protected void registerNamespaces(final Map<String, String> nsContext) { protected void registerNamespaces(final Map<String, String> nsContext) {

10
dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/migrate_mongo_mstores_parameters.json
View File

@ -48,21 +48,21 @@
"paramRequired": true "paramRequired": true
}, },
{ {
"paramName": "postgresUrl", "paramName": "pgurl",
"paramLongName": "postgresUrl", "paramLongName": "postgresUrl",
"paramDescription": "postgres url, example: jdbc:postgresql://localhost:5432/testdb", "paramDescription": "postgres url, example: jdbc:postgresql://localhost:5432/testdb",
"paramRequired": true "paramRequired": true
}, },
{ {
"paramName": "postgresUser", "paramName": "pguser",
"paramLongName": "postgresUser", "paramLongName": "postgresUser",
"paramDescription": "postgres user", "paramDescription": "postgres user",
"paramRequired": true "paramRequired": false
}, },
{ {
"paramName": "postgresPassword", "paramName": "pgpasswd",
"paramLongName": "postgresPassword", "paramLongName": "postgresPassword",
"paramDescription": "postgres password", "paramDescription": "postgres password",
"paramRequired": true "paramRequired": false
} }
] ]