From 198acac6b204acd38a2fb5572444b791072509d7 Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Mon, 18 Apr 2022 18:38:22 +0300 Subject: [PATCH] Add background scheduled tasks in order to delete unused files (when and how frequent they run is configurable) --- .../file/FileTimeConfiguration.java | 9 ++ .../configurations/file/PermProperties.java | 17 +++ .../file/StartTimeProperties.java | 59 +++++++++ .../configurations/file/TimeProperties.java | 33 +++++ .../configurations/file/TmpProperties.java | 17 +++ .../eu/eudat/logic/managers/FileManager.java | 8 +- .../schedule/file/FileCleanUpJobs.java | 120 ++++++++++++++++++ .../src/main/resources/config/application.yml | 20 ++- 8 files changed, 279 insertions(+), 4 deletions(-) create mode 100644 dmp-backend/web/src/main/java/eu/eudat/configurations/file/FileTimeConfiguration.java create mode 100644 dmp-backend/web/src/main/java/eu/eudat/configurations/file/PermProperties.java create mode 100644 dmp-backend/web/src/main/java/eu/eudat/configurations/file/StartTimeProperties.java create mode 100644 dmp-backend/web/src/main/java/eu/eudat/configurations/file/TimeProperties.java create mode 100644 dmp-backend/web/src/main/java/eu/eudat/configurations/file/TmpProperties.java create mode 100644 dmp-backend/web/src/main/java/eu/eudat/logic/utilities/schedule/file/FileCleanUpJobs.java diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/file/FileTimeConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/file/FileTimeConfiguration.java new file mode 100644 index 000000000..f1e26f15c --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/file/FileTimeConfiguration.java @@ -0,0 +1,9 @@ +package eu.eudat.configurations.file; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EnableConfigurationProperties({TmpProperties.class, PermProperties.class}) +public class FileTimeConfiguration { +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/file/PermProperties.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/file/PermProperties.java new file mode 100644 index 000000000..ab33c20bc --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/file/PermProperties.java @@ -0,0 +1,17 @@ +package eu.eudat.configurations.file; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "perm") +public class PermProperties { + + private TimeProperties time; + + public TimeProperties getTime() { + return time; + } + + public void setTime(TimeProperties time) { + this.time = time; + } +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/file/StartTimeProperties.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/file/StartTimeProperties.java new file mode 100644 index 000000000..e403bb072 --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/file/StartTimeProperties.java @@ -0,0 +1,59 @@ +package eu.eudat.configurations.file; + +public class StartTimeProperties { + private Integer month; + private String dayOfWeekName; + private Integer dayOfWeek; + + private Integer dayOfMonth; + private Integer hour; + private Integer minute; + + public Integer getMonth() { + return month; + } + + public void setMonth(Integer month) { + this.month = month; + } + + public String getDayOfWeekName() { + return dayOfWeekName; + } + + public void setDayOfWeekName(String dayOfWeekName) { + this.dayOfWeekName = dayOfWeekName; + } + + public Integer getDayOfWeek() { + return dayOfWeek; + } + + public void setDayOfWeek(Integer dayOfWeek) { + this.dayOfWeek = dayOfWeek; + } + + public Integer getDayOfMonth() { + return dayOfMonth; + } + + public void setDayOfMonth(Integer dayOfMonth) { + this.dayOfMonth = dayOfMonth; + } + + public Integer getHour() { + return hour; + } + + public void setHour(Integer hour) { + this.hour = hour; + } + + public Integer getMinute() { + return minute; + } + + public void setMinute(Integer minute) { + this.minute = minute; + } +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/file/TimeProperties.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/file/TimeProperties.java new file mode 100644 index 000000000..24bb3ad8d --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/file/TimeProperties.java @@ -0,0 +1,33 @@ +package eu.eudat.configurations.file; + +import java.time.temporal.ChronoUnit; + +public class TimeProperties { + private Integer interval; + private ChronoUnit timeUnit; + private StartTimeProperties startTime; + + public Integer getInterval() { + return interval; + } + + public void setInterval(Integer interval) { + this.interval = interval; + } + + public ChronoUnit getTimeUnit() { + return timeUnit; + } + + public void setTimeUnit(ChronoUnit timeUnit) { + this.timeUnit = timeUnit; + } + + public StartTimeProperties getStartTime() { + return startTime; + } + + public void setStartTime(StartTimeProperties startTime) { + this.startTime = startTime; + } +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/file/TmpProperties.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/file/TmpProperties.java new file mode 100644 index 000000000..2700bdd26 --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/file/TmpProperties.java @@ -0,0 +1,17 @@ +package eu.eudat.configurations.file; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "tmp") +public class TmpProperties { + + private TimeProperties time; + + public TimeProperties getTime() { + return time; + } + + public void setTime(TimeProperties time) { + this.time = time; + } +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FileManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FileManager.java index cb65ffa34..d1ddd352c 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FileManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FileManager.java @@ -60,6 +60,10 @@ public class FileManager { databaseRepository.getFileUploadDao().createOrUpdate(fileUpload); } + public List getDeletedFileUploads() { + return databaseRepository.getFileUploadDao().asQueryable().where(((builder, root) -> builder.equal(root.get("isDeleted"), true))).toList(); + } + public List getFileUploadsForEntityId(String entityId) { return databaseRepository.getFileUploadDao().asQueryable() .where((builder, root) -> builder.equal(root.get("entityId"), entityId)).toList(); @@ -74,9 +78,7 @@ public class FileManager { public void markAllFilesOfEntityIdAsDeleted(UUID entityId) { List fileUploads = this.getCurrentFileUploadsForEntityId(entityId); - fileUploads.forEach(fileUpload -> { - this.markOldFileAsDeleted(fileUpload); - }); + fileUploads.forEach(this::markOldFileAsDeleted); } public void createFile(String id, String fileName, String fileType, String entityId, FileUpload.EntityType entityType, UserInfo userInfo) { diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/schedule/file/FileCleanUpJobs.java b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/schedule/file/FileCleanUpJobs.java new file mode 100644 index 000000000..2fbcd0db4 --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/schedule/file/FileCleanUpJobs.java @@ -0,0 +1,120 @@ +package eu.eudat.logic.utilities.schedule.file; + +import eu.eudat.configurations.file.PermProperties; +import eu.eudat.configurations.file.TimeProperties; +import eu.eudat.configurations.file.TmpProperties; +import eu.eudat.data.entities.FileUpload; +import eu.eudat.logic.managers.FileManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +@Component +public class FileCleanUpJobs { + private static final Logger logger = LoggerFactory.getLogger(FileCleanUpJobs.class); + + private final FileManager fileManager; + private final Environment environment; + + @Autowired + public FileCleanUpJobs(FileManager fileManager, Environment environment, PermProperties permProperties, TmpProperties tmpProperties) { + this.fileManager = fileManager; + this.environment = environment; + if (permProperties != null) { + System.getProperties().put("perm.cron", createCron(permProperties.getTime())); + } + if (tmpProperties != null) { + System.getProperties().put("temp.cron", createCron(tmpProperties.getTime())); + } + + } + + @Scheduled(cron = "${perm.cron}") + public void deleteUnusedPermFiles() { + logger.info("Start deleting unused permanent files"); + List fileUploads = fileManager.getDeletedFileUploads(); + try { + List tempFilePaths = Files.walk(Paths.get(Objects.requireNonNull(environment.getProperty("file.storage")))).filter(Files::isRegularFile).collect(Collectors.toList()); + for (Path tempFilePath : tempFilePaths) { + if (fileUploads.stream().map(FileUpload::getId).anyMatch(uuid -> tempFilePath.endsWith(uuid.toString()))) { + Files.deleteIfExists(tempFilePath); + } + } + logger.info("Unused permanent files have been successfully deleted"); + } catch (IOException e) { + logger.error(e.getLocalizedMessage(), e); + } + } + + @Scheduled(cron = "${temp.cron}") + public void deleteTempFiles() { + logger.info("Start deleting temporary files"); + try { + List tempFilePaths = Files.walk(Paths.get(Objects.requireNonNull(environment.getProperty("temp.temp")))).filter(Files::isRegularFile).collect(Collectors.toList()); + for (Path tempFilePath : tempFilePaths) { + Files.deleteIfExists(tempFilePath); + } + logger.info("Temporary files have been successfully deleted"); + } catch (IOException e) { + logger.error(e.getLocalizedMessage(), e); + } + } + + private String createCron(TimeProperties timeProperties) { + List cronList = new LinkedList<>(); + cronList.add("0"); + switch (timeProperties.getTimeUnit()) { + case MINUTES: + cronList.add("0/" + timeProperties.getInterval()); + break; + case HOURS: + cronList.add(toSafeString(timeProperties.getStartTime().getMinute())); + cronList.add("0/" + timeProperties.getInterval()); + break; + case DAYS: + cronList.add(toSafeString(timeProperties.getStartTime().getMinute())); + cronList.add(toSafeString(timeProperties.getStartTime().getHour())); + cronList.add("*/" + timeProperties.getInterval()); + break; + case MONTHS: + cronList.add(toSafeString(timeProperties.getStartTime().getMinute())); + cronList.add(toSafeString(timeProperties.getStartTime().getHour())); + cronList.add(toSafeString(timeProperties.getStartTime().getDayOfMonth(), "1")); + cronList.add("*/" + timeProperties.getInterval()); + break; + case WEEKS: + cronList.add(toSafeString(timeProperties.getStartTime().getMinute())); + cronList.add(toSafeString(timeProperties.getStartTime().getHour())); + cronList.add("*/" + (timeProperties.getInterval() * 7)); + cronList.add("*"); + cronList.add(toSafeString(timeProperties.getStartTime().getDayOfWeek(), "1")); + break; + } + if (cronList.size() < 6) { + for (int i = cronList.size(); i < 6; i++) { + cronList.add("*"); + } + } + return String.join(" ", cronList); + } + + private String toSafeString(Object object) { + return toSafeString(object, "0"); + } + + private String toSafeString(Object object, String defaultValue) { + return object != null ? object.toString() : defaultValue; + } +} diff --git a/dmp-backend/web/src/main/resources/config/application.yml b/dmp-backend/web/src/main/resources/config/application.yml index e21fd971e..d88fd3d90 100644 --- a/dmp-backend/web/src/main/resources/config/application.yml +++ b/dmp-backend/web/src/main/resources/config/application.yml @@ -219,4 +219,22 @@ contact_email: #############CONFIRMATION EMAIL CONFIGURATIONS######### conf_email: expiration_time_seconds: 14400 - subject: "OpenDMP email confirmation" \ No newline at end of file + subject: "OpenDMP email confirmation" + +#############File Cleanup######### +tmp: + time: + interval: 1 + time-unit: WEEKS + start-time: + day-of-week: 4 + hour: 2 + minute: 15 +perm: + time: + interval: 1 + time-unit: WEEKS + start-time: + day-of-week: 4 + hour: 3 + minute: 15 \ No newline at end of file