Refs #10677: Force use of fallback causes duplication of records in fallback files
Task-Url: https://support.d4science.org/issues/10677 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/document-store-lib@160446 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
51840ce06d
commit
51f952c441
16
pom.xml
16
pom.xml
|
@ -32,18 +32,27 @@
|
||||||
<developerConnection>scm:https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/${project.artifactId}</developerConnection>
|
<developerConnection>scm:https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/${project.artifactId}</developerConnection>
|
||||||
<url>https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/${project.artifactId}</url>
|
<url>https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/${project.artifactId}</url>
|
||||||
</scm>
|
</scm>
|
||||||
|
|
||||||
|
<dependencyManagement>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.distribution</groupId>
|
||||||
|
<artifactId>gcube-bom</artifactId>
|
||||||
|
<version>LATEST</version>
|
||||||
|
<type>pom</type>
|
||||||
|
<scope>import</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</dependencyManagement>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.slf4j</groupId>
|
<groupId>org.slf4j</groupId>
|
||||||
<artifactId>slf4j-api</artifactId>
|
<artifactId>slf4j-api</artifactId>
|
||||||
<version>1.7.5</version>
|
|
||||||
<scope>provided</scope>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.fasterxml.jackson.core</groupId>
|
<groupId>com.fasterxml.jackson.core</groupId>
|
||||||
<artifactId>jackson-databind</artifactId>
|
<artifactId>jackson-databind</artifactId>
|
||||||
<version>2.6.0</version>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<!-- Test Dependencies -->
|
<!-- Test Dependencies -->
|
||||||
<dependency>
|
<dependency>
|
||||||
|
@ -59,7 +68,6 @@
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
<plugins>
|
<plugins>
|
||||||
<plugin>
|
<plugin>
|
||||||
|
|
|
@ -5,9 +5,7 @@ package org.gcube.documentstore.persistence;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
import java.io.FileReader;
|
import java.io.FileReader;
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
@ -46,79 +44,71 @@ public class FallbackMonitor implements Runnable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void elaborateFile(File elaborationFile){
|
protected void elaborateFile(File elaborationFile) {
|
||||||
try(BufferedReader br = new BufferedReader(new FileReader(elaborationFile))) {
|
try(BufferedReader br = new BufferedReader(new FileReader(elaborationFile))) {
|
||||||
for(String line; (line = br.readLine()) != null; ) {
|
for(String line; (line = br.readLine()) != null; ) {
|
||||||
try {
|
try {
|
||||||
Record record = RecordUtility.getRecord(line);
|
Record record = RecordUtility.getRecord(line);
|
||||||
persistenceBackend.accountWithFallback(record);
|
persistenceBackend.accountWithFallback(record);
|
||||||
} catch(Exception e){
|
} catch(Throwable e){
|
||||||
logger.error("Was not possible parse line {} to obtain a valid Record. Going to writing back this line as string fallback file.", line, e);
|
logger.error("Was not possible parse line {} to obtain a valid Record. Going to writing back this line as string fallback file.", line, e);
|
||||||
FallbackPersistenceBackend fallbackPersistenceBackend = persistenceBackend.getFallbackPersistence();
|
FallbackPersistenceBackend fallbackPersistenceBackend = persistenceBackend.getFallbackPersistence();
|
||||||
try {
|
try {
|
||||||
fallbackPersistenceBackend.printLine(line);
|
fallbackPersistenceBackend.printLine(line);
|
||||||
} catch (Exception e1) {
|
} catch (Throwable e1) {
|
||||||
logger.error("Line {} will be lost", line, e1);
|
logger.error("Line {} will be lost", line, e1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (FileNotFoundException e) {
|
} catch (Throwable e) {
|
||||||
logger.error("File non trovato", e);
|
logger.error("Error elaborating {}", elaborationFile.getAbsoluteFile(), e);
|
||||||
} catch (IOException e) {
|
|
||||||
logger.error("IOException", e);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Deprecated
|
@Deprecated
|
||||||
protected void manageOldAccountingFile(){
|
protected void manageOldAccountingFile(){
|
||||||
FallbackPersistenceBackend fallbackPersistenceBackend = persistenceBackend.getFallbackPersistence();
|
FallbackPersistenceBackend fallbackPersistenceBackend = persistenceBackend.getFallbackPersistence();
|
||||||
File newFile = fallbackPersistenceBackend.getFallbackFile();
|
elaborateFallbackFile(fallbackPersistenceBackend.getOldFallbackFile());
|
||||||
|
|
||||||
String oldAccountingFileName = newFile.getName();
|
|
||||||
int lastIndexOf_ = oldAccountingFileName.lastIndexOf("_");
|
|
||||||
|
|
||||||
oldAccountingFileName = oldAccountingFileName.substring(lastIndexOf_+1);
|
|
||||||
oldAccountingFileName = oldAccountingFileName.replace(
|
|
||||||
PersistenceBackendFactory.FALLBACK_FILENAME, "accountingFallback.log");
|
|
||||||
|
|
||||||
|
|
||||||
File oldAccountingFile = new File(newFile.getParentFile(), oldAccountingFileName);
|
|
||||||
elaborateFallbackFile(oldAccountingFile);
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected synchronized void elaborateFallbackFile(File file){
|
protected void elaborateFallbackFile(File elaborationFile){
|
||||||
logger.trace("Trying to persist {}s which failed and were persisted using fallback on file {}",
|
if(elaborationFile!=null && elaborationFile.exists()){
|
||||||
Record.class.getSimpleName(), file.getAbsoluteFile());
|
try {
|
||||||
File elaborationFile = null;
|
elaborateFile(elaborationFile);
|
||||||
|
}finally {
|
||||||
if(file.exists()){
|
if(elaborationFile.exists()) {
|
||||||
Long timestamp = Calendar.getInstance().getTimeInMillis();
|
boolean deleted = elaborationFile.delete();
|
||||||
elaborationFile = new File(file.getAbsolutePath() + ELABORATION_FILE_SUFFIX + "." + timestamp.toString());
|
if(!deleted){
|
||||||
logger.trace("Going to move fallaback file ({}) to elaboration file ({})",
|
logger.error("Failed to delete file {}", elaborationFile.getAbsolutePath());
|
||||||
file.getAbsolutePath(), elaborationFile.getAbsolutePath());
|
File elaborationFileNotDeleted = new File(elaborationFile.getAbsolutePath()+ELABORATION_FILE_NOT_DELETED_SUFFIX);
|
||||||
file.renameTo(elaborationFile);
|
elaborationFile.renameTo(elaborationFileNotDeleted);
|
||||||
|
}
|
||||||
}
|
}else {
|
||||||
|
logger.error("File {} does not exists. This is really starge and should not occur. Please contact the administrator.", elaborationFile.getAbsolutePath());
|
||||||
if(elaborationFile!=null){
|
}
|
||||||
elaborateFile(elaborationFile);
|
|
||||||
boolean deleted = elaborationFile.delete();
|
|
||||||
if(!deleted){
|
|
||||||
logger.trace("Failed to delete file {}", elaborationFile.getAbsolutePath());
|
|
||||||
File elaborationFileNotDeleted = new File(elaborationFile.getAbsolutePath()+ELABORATION_FILE_NOT_DELETED_SUFFIX);
|
|
||||||
elaborationFile.renameTo(elaborationFileNotDeleted);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void elaborateFallbackFile(){
|
||||||
|
FallbackPersistenceBackend fallbackPersistenceBackend = persistenceBackend.getFallbackPersistence();
|
||||||
|
File file = fallbackPersistenceBackend.getFallbackFile();
|
||||||
|
|
||||||
|
logger.trace("Trying to persist {}s which were persisted using fallback on file {}",
|
||||||
|
Record.class.getSimpleName(), file.getAbsoluteFile());
|
||||||
|
|
||||||
|
Long timestamp = Calendar.getInstance().getTimeInMillis();
|
||||||
|
File elaborationFile = fallbackPersistenceBackend.moveFallbackFile(ELABORATION_FILE_SUFFIX + "." + timestamp.toString());
|
||||||
|
|
||||||
|
elaborateFallbackFile(elaborationFile);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
//logger.trace("TestingThread PersistenceBackendMonitor run");
|
elaborateFallbackFile();
|
||||||
FallbackPersistenceBackend fallbackPersistenceBackend = persistenceBackend.getFallbackPersistence();
|
|
||||||
File file = new File(fallbackPersistenceBackend.getFallbackFile().getAbsolutePath());
|
|
||||||
elaborateFallbackFile(file);
|
|
||||||
manageOldAccountingFile();
|
manageOldAccountingFile();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,9 @@ import java.io.File;
|
||||||
import java.io.FileWriter;
|
import java.io.FileWriter;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.PrintWriter;
|
import java.io.PrintWriter;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.StandardCopyOption;
|
||||||
|
|
||||||
import org.gcube.documentstore.records.DSMapper;
|
import org.gcube.documentstore.records.DSMapper;
|
||||||
import org.gcube.documentstore.records.Record;
|
import org.gcube.documentstore.records.Record;
|
||||||
|
@ -27,7 +30,45 @@ public class FallbackPersistenceBackend extends PersistenceBackend {
|
||||||
* @return the fallbackFile
|
* @return the fallbackFile
|
||||||
*/
|
*/
|
||||||
protected File getFallbackFile() {
|
protected File getFallbackFile() {
|
||||||
return fallbackFile;
|
synchronized(fallbackFile) {
|
||||||
|
return fallbackFile;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Deprecated
|
||||||
|
protected File getOldFallbackFile() {
|
||||||
|
synchronized(fallbackFile) {
|
||||||
|
String oldAccountingFileName = fallbackFile.getName();
|
||||||
|
int lastIndexOf_ = oldAccountingFileName.lastIndexOf("_");
|
||||||
|
|
||||||
|
oldAccountingFileName = oldAccountingFileName.substring(lastIndexOf_+1);
|
||||||
|
oldAccountingFileName = oldAccountingFileName.replace(
|
||||||
|
PersistenceBackendFactory.FALLBACK_FILENAME, "accountingFallback.log");
|
||||||
|
|
||||||
|
return new File(fallbackFile.getParentFile(), oldAccountingFileName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Move the fallbackFile to a new file with the same name by appending a suffix
|
||||||
|
* @param suffix
|
||||||
|
* @return the move file
|
||||||
|
*/
|
||||||
|
protected File moveFallbackFile(String suffix) {
|
||||||
|
synchronized(fallbackFile) {
|
||||||
|
try {
|
||||||
|
Path source = fallbackFile.toPath();
|
||||||
|
Path target = source.resolveSibling(fallbackFile.getName()+suffix);
|
||||||
|
logger.trace("Going to move fallback file {} to {}", source.toAbsolutePath().toString(), target.toAbsolutePath().toString());
|
||||||
|
target = Files.move(source, source.resolveSibling(fallbackFile.getName()+suffix), StandardCopyOption.ATOMIC_MOVE);
|
||||||
|
return target.toFile();
|
||||||
|
} catch (Exception e) {
|
||||||
|
String absPath = fallbackFile.getAbsolutePath();
|
||||||
|
String error = String.format("It was not possibile to move %s to %s%s.", absPath, absPath, suffix);
|
||||||
|
logger.error(error, e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected FallbackPersistenceBackend(File fallbackFile) {
|
protected FallbackPersistenceBackend(File fallbackFile) {
|
||||||
|
@ -56,23 +97,42 @@ public class FallbackPersistenceBackend extends PersistenceBackend {
|
||||||
// Nothing TO DO
|
// Nothing TO DO
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void accountWithFallback(Record... records) throws Exception {
|
||||||
|
for (Record record : records) {
|
||||||
|
try {
|
||||||
|
reallyAccount(record);
|
||||||
|
logger.trace("{} accounted succesfully from {}", record.toString(), FallbackPersistenceBackend.class.getSimpleName());
|
||||||
|
} catch (Throwable th) {
|
||||||
|
logger.error("{} was not accounted at all", record.toString(), th);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
* This method is synchronized on {@link File} used, so any actions which
|
|
||||||
* has to modify, rename or delete the file must be synchronized on this
|
|
||||||
* file. To retrieve it use {@link #getFallbackFile()} method.
|
|
||||||
* This is intended for internal library usage only so that is protected
|
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
protected void reallyAccount(Record record) throws Exception {
|
protected void reallyAccount(Record record) {
|
||||||
String marshalled = DSMapper.marshal(record);
|
String marshalled = null;
|
||||||
logger.debug("reallyAccount:{}",marshalled);
|
try {
|
||||||
printLine(marshalled);
|
marshalled = DSMapper.marshal(record);
|
||||||
|
printLine(marshalled);
|
||||||
|
logger.trace("{} accounted succesfully from {}", marshalled, this.getClass().getSimpleName());
|
||||||
|
} catch(Throwable th) {
|
||||||
|
logger.error("{} was not accounted at all", marshalled!=null ? marshalled : record.toString(), th);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void printLine(String line) throws Exception {
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
* This method is synchronized on fallbackFile. Any actions which
|
||||||
|
* has to modify, rename or delete the file must be synchronized on this
|
||||||
|
* file. To retrieve it use {@link #getFallbackFile()} method.
|
||||||
|
* This is intended for internal library usage only so that is protected.
|
||||||
|
* It
|
||||||
|
*/
|
||||||
|
protected void printLine(String line) throws Exception {
|
||||||
synchronized (fallbackFile) {
|
synchronized (fallbackFile) {
|
||||||
try(FileWriter fw = new FileWriter(fallbackFile, true);
|
try(FileWriter fw = new FileWriter(fallbackFile, true);
|
||||||
BufferedWriter bw = new BufferedWriter(fw);
|
BufferedWriter bw = new BufferedWriter(fw);
|
||||||
|
|
|
@ -141,6 +141,63 @@ public abstract class PersistenceBackend {
|
||||||
*/
|
*/
|
||||||
protected abstract void reallyAccount(Record record) throws Exception;
|
protected abstract void reallyAccount(Record record) throws Exception;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The function check if the use of fallback is forced and use it if any.
|
||||||
|
* The function always return the value of forceFallbackUse even if after using it otherwise the record is
|
||||||
|
* accounted two times in case the forced time is terminated
|
||||||
|
* @param record the rEcord to account with fallback if any
|
||||||
|
* @return
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
private synchronized boolean isFallbackForced() throws Exception {
|
||||||
|
if (forceFallbackUse) {
|
||||||
|
String fallbackPersistenceName = FallbackPersistenceBackend.class.getSimpleName();
|
||||||
|
|
||||||
|
long now = Calendar.getInstance().getTimeInMillis();
|
||||||
|
long diff = now - fallbackUseStartTime;
|
||||||
|
logger.trace("{} forced use started at {}. {} seconds were elapsed",
|
||||||
|
fallbackPersistenceName, fallbackUseStartTime, (long) diff/1000);
|
||||||
|
|
||||||
|
if (diff > MAX_TIME_TO_FALLBACK) {
|
||||||
|
logger.info("The time to force the usage of {} is terminated. Trying to restore the use of {}",
|
||||||
|
fallbackPersistenceName, this.getClass().getSimpleName());
|
||||||
|
forceFallbackUse = false;
|
||||||
|
fallbackUseCounter = 0;
|
||||||
|
fallbackUseStartTime = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
return forceFallbackUse;
|
||||||
|
}
|
||||||
|
|
||||||
|
private synchronized void registerUseOfFallback() throws Exception {
|
||||||
|
/*
|
||||||
|
* This if could be removed because the function id only used from accountWithFallback() which is overrode
|
||||||
|
* in FallbackPersistenceBackend.
|
||||||
|
* Anyway it is keep to avoid future error if the function will be used in others functions.
|
||||||
|
* */
|
||||||
|
if (!(this instanceof FallbackPersistenceBackend) ){
|
||||||
|
String fallbackPersistenceName = FallbackPersistenceBackend.class.getSimpleName();
|
||||||
|
|
||||||
|
fallbackUseCounter++;
|
||||||
|
|
||||||
|
logger.trace("Exception number is {}. Max Retry number is {}. After that the use of {} will be forced",
|
||||||
|
fallbackUseCounter, MAX_FALLBACK_RETRY, fallbackPersistenceName);
|
||||||
|
|
||||||
|
if (fallbackUseCounter == MAX_FALLBACK_RETRY) {
|
||||||
|
forceFallbackUse = true;
|
||||||
|
fallbackUseStartTime = Calendar.getInstance().getTimeInMillis();
|
||||||
|
logger.info("Going to force {} for too many Exceptions", fallbackPersistenceName);
|
||||||
|
|
||||||
|
// aggregationScheduler.flush(new DefaultPersitenceExecutor(fallbackPersistence));
|
||||||
|
|
||||||
|
this.close();
|
||||||
|
this.clean();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/***
|
/***
|
||||||
*
|
*
|
||||||
* @param records
|
* @param records
|
||||||
|
@ -157,62 +214,23 @@ public abstract class PersistenceBackend {
|
||||||
try {
|
try {
|
||||||
recordString = record.toString();
|
recordString = record.toString();
|
||||||
|
|
||||||
if (forceFallbackUse) {
|
if (isFallbackForced()) {
|
||||||
|
logger.trace("Forcing the use of {} to account {}", fallbackPersistenceName, record.toString());
|
||||||
logger.trace("Forcing the use of {} to account {}", fallbackPersistenceName, recordString);
|
|
||||||
fallbackPersistence.reallyAccount(record);
|
fallbackPersistence.reallyAccount(record);
|
||||||
|
|
||||||
long now = Calendar.getInstance().getTimeInMillis();
|
|
||||||
long diff = now - fallbackUseStartTime;
|
|
||||||
logger.trace("{} forced use started at {}. {} seconds were elapsed",
|
|
||||||
fallbackPersistenceName, fallbackUseStartTime, (long) diff/1000);
|
|
||||||
|
|
||||||
if (diff > MAX_TIME_TO_FALLBACK) {
|
|
||||||
logger.debug("The time to force the usage of {} is terminated. Trying to restore the use of {}",
|
|
||||||
fallbackPersistenceName, persistenceName);
|
|
||||||
forceFallbackUse = false;
|
|
||||||
fallbackUseCounter = 0;
|
|
||||||
fallbackUseStartTime = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
this.reallyAccount(record);
|
this.reallyAccount(record);
|
||||||
logger.trace("{} accounted succesfully from {}.", record.toString(), persistenceName);
|
logger.trace("{} accounted succesfully from {}.", recordString, persistenceName);
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (Throwable t) {
|
} catch (Throwable t) {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
logger.warn("{} was not accounted succesfully using {}. Trying to use {}.", recordString,
|
logger.warn("{} was not accounted succesfully using {}. Trying to use {}.", recordString,
|
||||||
persistenceName, fallbackPersistenceName, t);
|
persistenceName, fallbackPersistenceName, t);
|
||||||
|
|
||||||
fallbackPersistence.reallyAccount(record);
|
fallbackPersistence.reallyAccount(record);
|
||||||
|
}finally {
|
||||||
logger.trace("{} accounted succesfully from {}", recordString, fallbackPersistenceName);
|
registerUseOfFallback();
|
||||||
} catch (Throwable th) {
|
|
||||||
logger.error("{} was not accounted at all", recordString, t);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!(this instanceof FallbackPersistenceBackend) ){
|
|
||||||
// && (t.getCause() != null && t.getCause() instanceof TimeoutException)) {
|
|
||||||
|
|
||||||
fallbackUseCounter++;
|
|
||||||
|
|
||||||
logger.warn("Exception number is {}. Max Retry number is {}. After that the use of {} will be forced",
|
|
||||||
fallbackUseCounter, MAX_FALLBACK_RETRY, fallbackPersistenceName);
|
|
||||||
|
|
||||||
if (fallbackUseCounter == MAX_FALLBACK_RETRY) {
|
|
||||||
forceFallbackUse = true;
|
|
||||||
fallbackUseStartTime = Calendar.getInstance().getTimeInMillis();
|
|
||||||
logger.trace("Going to force {} for too many Exceptions",
|
|
||||||
fallbackPersistenceName);
|
|
||||||
|
|
||||||
aggregationScheduler.flush(new DefaultPersitenceExecutor(fallbackPersistence));
|
|
||||||
|
|
||||||
this.close();
|
|
||||||
this.clean();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue