Add background scheduled tasks in order to delete unused files (when and how frequent they run is configurable)

update-configs
George Kalampokis 2 years ago
parent 85b37dd16d
commit b012f415a7

@ -0,0 +1,9 @@
package eu.eudat.configurations.file;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties({TmpProperties.class, PermProperties.class})
public class FileTimeConfiguration {
}

@ -0,0 +1,17 @@
package eu.eudat.configurations.file;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "perm")
public class PermProperties {
private TimeProperties time;
public TimeProperties getTime() {
return time;
}
public void setTime(TimeProperties time) {
this.time = time;
}
}

@ -0,0 +1,59 @@
package eu.eudat.configurations.file;
public class StartTimeProperties {
private Integer month;
private String dayOfWeekName;
private Integer dayOfWeek;
private Integer dayOfMonth;
private Integer hour;
private Integer minute;
public Integer getMonth() {
return month;
}
public void setMonth(Integer month) {
this.month = month;
}
public String getDayOfWeekName() {
return dayOfWeekName;
}
public void setDayOfWeekName(String dayOfWeekName) {
this.dayOfWeekName = dayOfWeekName;
}
public Integer getDayOfWeek() {
return dayOfWeek;
}
public void setDayOfWeek(Integer dayOfWeek) {
this.dayOfWeek = dayOfWeek;
}
public Integer getDayOfMonth() {
return dayOfMonth;
}
public void setDayOfMonth(Integer dayOfMonth) {
this.dayOfMonth = dayOfMonth;
}
public Integer getHour() {
return hour;
}
public void setHour(Integer hour) {
this.hour = hour;
}
public Integer getMinute() {
return minute;
}
public void setMinute(Integer minute) {
this.minute = minute;
}
}

@ -0,0 +1,33 @@
package eu.eudat.configurations.file;
import java.time.temporal.ChronoUnit;
public class TimeProperties {
private Integer interval;
private ChronoUnit timeUnit;
private StartTimeProperties startTime;
public Integer getInterval() {
return interval;
}
public void setInterval(Integer interval) {
this.interval = interval;
}
public ChronoUnit getTimeUnit() {
return timeUnit;
}
public void setTimeUnit(ChronoUnit timeUnit) {
this.timeUnit = timeUnit;
}
public StartTimeProperties getStartTime() {
return startTime;
}
public void setStartTime(StartTimeProperties startTime) {
this.startTime = startTime;
}
}

@ -0,0 +1,17 @@
package eu.eudat.configurations.file;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "tmp")
public class TmpProperties {
private TimeProperties time;
public TimeProperties getTime() {
return time;
}
public void setTime(TimeProperties time) {
this.time = time;
}
}

@ -60,6 +60,10 @@ public class FileManager {
databaseRepository.getFileUploadDao().createOrUpdate(fileUpload);
}
public List<FileUpload> getDeletedFileUploads() {
return databaseRepository.getFileUploadDao().asQueryable().where(((builder, root) -> builder.equal(root.get("isDeleted"), true))).toList();
}
public List<FileUpload> getFileUploadsForEntityId(String entityId) {
return databaseRepository.getFileUploadDao().asQueryable()
.where((builder, root) -> builder.equal(root.get("entityId"), entityId)).toList();
@ -74,9 +78,7 @@ public class FileManager {
public void markAllFilesOfEntityIdAsDeleted(UUID entityId) {
List<FileUpload> fileUploads = this.getCurrentFileUploadsForEntityId(entityId);
fileUploads.forEach(fileUpload -> {
this.markOldFileAsDeleted(fileUpload);
});
fileUploads.forEach(this::markOldFileAsDeleted);
}
public void createFile(String id, String fileName, String fileType, String entityId, FileUpload.EntityType entityType, UserInfo userInfo) {

@ -0,0 +1,120 @@
package eu.eudat.logic.utilities.schedule.file;
import eu.eudat.configurations.file.PermProperties;
import eu.eudat.configurations.file.TimeProperties;
import eu.eudat.configurations.file.TmpProperties;
import eu.eudat.data.entities.FileUpload;
import eu.eudat.logic.managers.FileManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@Component
public class FileCleanUpJobs {
private static final Logger logger = LoggerFactory.getLogger(FileCleanUpJobs.class);
private final FileManager fileManager;
private final Environment environment;
@Autowired
public FileCleanUpJobs(FileManager fileManager, Environment environment, PermProperties permProperties, TmpProperties tmpProperties) {
this.fileManager = fileManager;
this.environment = environment;
if (permProperties != null) {
System.getProperties().put("perm.cron", createCron(permProperties.getTime()));
}
if (tmpProperties != null) {
System.getProperties().put("temp.cron", createCron(tmpProperties.getTime()));
}
}
@Scheduled(cron = "${perm.cron}")
public void deleteUnusedPermFiles() {
logger.info("Start deleting unused permanent files");
List<FileUpload> fileUploads = fileManager.getDeletedFileUploads();
try {
List<Path> tempFilePaths = Files.walk(Paths.get(Objects.requireNonNull(environment.getProperty("file.storage")))).filter(Files::isRegularFile).collect(Collectors.toList());
for (Path tempFilePath : tempFilePaths) {
if (fileUploads.stream().map(FileUpload::getId).anyMatch(uuid -> tempFilePath.endsWith(uuid.toString()))) {
Files.deleteIfExists(tempFilePath);
}
}
logger.info("Unused permanent files have been successfully deleted");
} catch (IOException e) {
logger.error(e.getLocalizedMessage(), e);
}
}
@Scheduled(cron = "${temp.cron}")
public void deleteTempFiles() {
logger.info("Start deleting temporary files");
try {
List<Path> tempFilePaths = Files.walk(Paths.get(Objects.requireNonNull(environment.getProperty("temp.temp")))).filter(Files::isRegularFile).collect(Collectors.toList());
for (Path tempFilePath : tempFilePaths) {
Files.deleteIfExists(tempFilePath);
}
logger.info("Temporary files have been successfully deleted");
} catch (IOException e) {
logger.error(e.getLocalizedMessage(), e);
}
}
private String createCron(TimeProperties timeProperties) {
List<String> cronList = new LinkedList<>();
cronList.add("0");
switch (timeProperties.getTimeUnit()) {
case MINUTES:
cronList.add("0/" + timeProperties.getInterval());
break;
case HOURS:
cronList.add(toSafeString(timeProperties.getStartTime().getMinute()));
cronList.add("0/" + timeProperties.getInterval());
break;
case DAYS:
cronList.add(toSafeString(timeProperties.getStartTime().getMinute()));
cronList.add(toSafeString(timeProperties.getStartTime().getHour()));
cronList.add("*/" + timeProperties.getInterval());
break;
case MONTHS:
cronList.add(toSafeString(timeProperties.getStartTime().getMinute()));
cronList.add(toSafeString(timeProperties.getStartTime().getHour()));
cronList.add(toSafeString(timeProperties.getStartTime().getDayOfMonth(), "1"));
cronList.add("*/" + timeProperties.getInterval());
break;
case WEEKS:
cronList.add(toSafeString(timeProperties.getStartTime().getMinute()));
cronList.add(toSafeString(timeProperties.getStartTime().getHour()));
cronList.add("*/" + (timeProperties.getInterval() * 7));
cronList.add("*");
cronList.add(toSafeString(timeProperties.getStartTime().getDayOfWeek(), "1"));
break;
}
if (cronList.size() < 6) {
for (int i = cronList.size(); i < 6; i++) {
cronList.add("*");
}
}
return String.join(" ", cronList);
}
private String toSafeString(Object object) {
return toSafeString(object, "0");
}
private String toSafeString(Object object, String defaultValue) {
return object != null ? object.toString() : defaultValue;
}
}

@ -219,4 +219,22 @@ contact_email:
#############CONFIRMATION EMAIL CONFIGURATIONS#########
conf_email:
expiration_time_seconds: 14400
subject: "OpenDMP email confirmation"
subject: "OpenDMP email confirmation"
#############File Cleanup#########
tmp:
time:
interval: 1
time-unit: WEEKS
start-time:
day-of-week: 4
hour: 2
minute: 15
perm:
time:
interval: 1
time-unit: WEEKS
start-time:
day-of-week: 4
hour: 3
minute: 15
Loading…
Cancel
Save