dimitris.pierrakos
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

adjusted logs

This commit is contained in:
Claudio Atzori 2023-03-01 11:58:07 +01:00
parent 16ad42e8f3
commit 7d263f265e
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateMongoMdstoresApplication.java
View File

@ -131,7 +131,7 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrationApplicatio
// that is the hdfs path basePath/MDSTOREID/timestamp is missing // that is the hdfs path basePath/MDSTOREID/timestamp is missing
// So we have to synch it // So we have to synch it
if (!hdfsMDStores.containsKey(currentMDStore.getMdstore())) { if (!hdfsMDStores.containsKey(currentMDStore.getMdstore())) {
log.info("Adding store " + currentMDStore.getMdstore()); log.info("Adding store {}", currentMDStore.getMdstore());
try { try {
synchMDStoreIntoHDFS( synchMDStoreIntoHDFS(
mdFormat, mdLayout, mdInterpretation, hdfsPath, fileSystem, mongoBaseUrl, mongoDb, mdFormat, mdLayout, mdInterpretation, hdfsPath, fileSystem, mongoBaseUrl, mongoDb,
@ -145,14 +145,14 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrationApplicatio
// basePath/MDSTOREID/timestamp but the timestamp on hdfs is older that the // basePath/MDSTOREID/timestamp but the timestamp on hdfs is older that the
// new one in mongo so we have to synch the new mdstore and delete the old one // new one in mongo so we have to synch the new mdstore and delete the old one
if (currentMDStore.getLatestTimestamp() > current.getLatestTimestamp()) { if (currentMDStore.getLatestTimestamp() > current.getLatestTimestamp()) {
log.info("Updating MDStore " + currentMDStore.getMdstore()); log.info("Updating MDStore {}", currentMDStore.getMdstore());
final String mdstoreDir = createMDStoreDir(hdfsPath, currentMDStore.getMdstore()); final String mdstoreDir = createMDStoreDir(hdfsPath, currentMDStore.getMdstore());
final String rmPath = createMDStoreDir(mdstoreDir, current.getLatestTimestamp().toString()); final String rmPath = createMDStoreDir(mdstoreDir, current.getLatestTimestamp().toString());
try { try {
synchMDStoreIntoHDFS( synchMDStoreIntoHDFS(
mdFormat, mdLayout, mdInterpretation, hdfsPath, fileSystem, mongoBaseUrl, mongoDb, mdFormat, mdLayout, mdInterpretation, hdfsPath, fileSystem, mongoBaseUrl, mongoDb,
currentMDStore); currentMDStore);
log.info("deleting " + rmPath); log.info("deleting {}", rmPath);
// DELETE THE OLD MDSTORE // DELETE THE OLD MDSTORE
fileSystem.delete(new Path(rmPath), true); fileSystem.delete(new Path(rmPath), true);
} catch (IOException e) { } catch (IOException e) {