D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 8 Projects 2 Releases 1 Wiki Activity

[stats wf] indicators across stats dbs & updates in the org ids #248

Closed
dimitris.pierrakos wants to merge 1742 commits from beta into beta2master_sept_2022
pull from: beta
merge into: D-Net:beta2master_sept_2022
Conversation 3 Commits 1742 Files Changed 1525 +3 -3
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 7d263f265e - Show all commits

6
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateMongoMdstoresApplication.java
View File

@ -131,7 +131,7 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrationApplicatio
// that is the hdfs path basePath/MDSTOREID/timestamp is missing // that is the hdfs path basePath/MDSTOREID/timestamp is missing
// So we have to synch it // So we have to synch it
if (!hdfsMDStores.containsKey(currentMDStore.getMdstore())) { if (!hdfsMDStores.containsKey(currentMDStore.getMdstore())) {
log.info("Adding store " + currentMDStore.getMdstore()); log.info("Adding store {}", currentMDStore.getMdstore());
try { try {
synchMDStoreIntoHDFS( synchMDStoreIntoHDFS(
mdFormat, mdLayout, mdInterpretation, hdfsPath, fileSystem, mongoBaseUrl, mongoDb, mdFormat, mdLayout, mdInterpretation, hdfsPath, fileSystem, mongoBaseUrl, mongoDb,
@ -145,14 +145,14 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrationApplicatio
// basePath/MDSTOREID/timestamp but the timestamp on hdfs is older that the // basePath/MDSTOREID/timestamp but the timestamp on hdfs is older that the
// new one in mongo so we have to synch the new mdstore and delete the old one // new one in mongo so we have to synch the new mdstore and delete the old one
if (currentMDStore.getLatestTimestamp() > current.getLatestTimestamp()) { if (currentMDStore.getLatestTimestamp() > current.getLatestTimestamp()) {
log.info("Updating MDStore " + currentMDStore.getMdstore()); log.info("Updating MDStore {}", currentMDStore.getMdstore());
final String mdstoreDir = createMDStoreDir(hdfsPath, currentMDStore.getMdstore()); final String mdstoreDir = createMDStoreDir(hdfsPath, currentMDStore.getMdstore());
final String rmPath = createMDStoreDir(mdstoreDir, current.getLatestTimestamp().toString()); final String rmPath = createMDStoreDir(mdstoreDir, current.getLatestTimestamp().toString());
try { try {
synchMDStoreIntoHDFS( synchMDStoreIntoHDFS(
mdFormat, mdLayout, mdInterpretation, hdfsPath, fileSystem, mongoBaseUrl, mongoDb, mdFormat, mdLayout, mdInterpretation, hdfsPath, fileSystem, mongoBaseUrl, mongoDb,
currentMDStore); currentMDStore);
log.info("deleting " + rmPath); log.info("deleting {}", rmPath);
// DELETE THE OLD MDSTORE // DELETE THE OLD MDSTORE
fileSystem.delete(new Path(rmPath), true); fileSystem.delete(new Path(rmPath), true);
} catch (IOException e) { } catch (IOException e) {