add storage

This commit is contained in:
Efstratios Giannopoulos 2023-11-28 17:23:56 +02:00
parent 1b92af8762
commit 337556265e
8 changed files with 665 additions and 1 deletions

2
.gitignore vendored
View File

@ -49,6 +49,6 @@ bin/
.run .run
openDMP/dmp-backend/uploads/ openDMP/dmp-backend/uploads/
openDMP/dmp-backend/tmp/ openDMP/dmp-backend/tmp/
storage/
logs/ logs/
dmp-backend/web/src/main/resources/certificates/ dmp-backend/web/src/main/resources/certificates/
/storage/

View File

@ -0,0 +1,13 @@
package eu.eudat.service.storage;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties(StorageFileCleanupProperties.class)
public class StorageFileCleanupConfiguration {
}

View File

@ -0,0 +1,30 @@
package eu.eudat.service.storage;
import eu.eudat.commons.enums.StorageType;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.List;
@ConfigurationProperties(prefix = "storage.task")
public class StorageFileCleanupProperties {
private Boolean enable;
private int intervalSeconds;
public Boolean getEnable() {
return enable;
}
public void setEnable(Boolean enable) {
this.enable = enable;
}
public int getIntervalSeconds() {
return intervalSeconds;
}
public void setIntervalSeconds(int intervalSeconds) {
this.intervalSeconds = intervalSeconds;
}
}

View File

@ -0,0 +1,198 @@
package eu.eudat.service.storage;
import eu.eudat.commons.fake.FakeRequestScope;
import eu.eudat.data.StorageFileEntity;
import eu.eudat.model.StorageFile;
import eu.eudat.query.StorageFileQuery;
import gr.cite.tools.data.query.Ordering;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import jakarta.persistence.EntityManagerFactory;
import jakarta.persistence.EntityTransaction;
import jakarta.persistence.OptimisticLockException;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationListener;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Service;
import java.io.Closeable;
import java.io.IOException;
import java.time.Instant;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Service
@Scope(value = ConfigurableBeanFactory.SCOPE_SINGLETON)
public class StorageFileCleanupTask implements Closeable, ApplicationListener<ApplicationReadyEvent> {
private class CandidateInfo
{
private UUID id;
private Instant createdAt;
public UUID getId() {
return id;
}
public void setId(UUID id) {
this.id = id;
}
public Instant getCreatedAt() {
return createdAt;
}
public void setCreatedAt(Instant createdAt) {
this.createdAt = createdAt;
}
}
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(StorageFileCleanupTask.class));
private final StorageFileCleanupProperties _config;
private final ApplicationContext applicationContext;
private ScheduledExecutorService scheduler;
public StorageFileCleanupTask(
StorageFileCleanupProperties config,
ApplicationContext applicationContext) {
this._config = config;
this.applicationContext = applicationContext;
}
@Override
public void onApplicationEvent(ApplicationReadyEvent event) {
long intervalSeconds = this._config .getIntervalSeconds();
if (this._config .getEnable() && intervalSeconds > 0) {
logger.info("File clean up run in {} seconds", intervalSeconds);
scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(this::process, 10, intervalSeconds, TimeUnit.SECONDS);
} else {
scheduler = null;
}
}
@Override
public void close() throws IOException {
if (scheduler != null) this.scheduler.close();
}
protected void process() {
if (!this._config.getEnable()) return;
try {
Instant lastCandidateCreationTimestamp = null;
while (true) {
CandidateInfo candidate = this.candidate(lastCandidateCreationTimestamp);
if (candidate == null) break;
lastCandidateCreationTimestamp = candidate.getCreatedAt();
logger.debug("Clean up file: {}", candidate.getId());
boolean successfullyProcessed = this.processStorageFile(candidate.getId());
if (!successfullyProcessed) {
logger.error("Problem processing file cleanups. {}", candidate.getId());
}
}
} catch (Exception ex) {
logger.error("Problem processing file cleanups. Breaking for next interval", ex);
}
}
private boolean processStorageFile(UUID fileId) {
EntityTransaction transaction = null;
EntityManager entityManager = null;
boolean success = false;
try (FakeRequestScope fakeRequestScope = new FakeRequestScope()) {
try {
QueryFactory queryFactory = this.applicationContext.getBean(QueryFactory.class);
EntityManagerFactory entityManagerFactory = this.applicationContext.getBean(EntityManagerFactory.class);
StorageFileService storageFileService = this.applicationContext.getBean(StorageFileService.class);
entityManager = entityManagerFactory.createEntityManager();
transaction = entityManager.getTransaction();
transaction.begin();
StorageFileEntity item = queryFactory.query(StorageFileQuery.class).ids(fileId).isPurged(false).first();
success = true;
if (item != null) {
storageFileService.purgeSafe(fileId);
}
transaction.commit();
} catch (OptimisticLockException ex) {
// we get this if/when someone else already modified the notifications. We want to essentially ignore this, and keep working
logger.debug("Concurrency exception getting file. Skipping: {} ", ex.getMessage());
if (transaction != null) transaction.rollback();
success = false;
} catch (Exception ex) {
logger.error("Problem getting list of file. Skipping: {}", ex.getMessage(), ex);
if (transaction != null) transaction.rollback();
success = false;
} finally {
if (entityManager != null) entityManager.close();
}
} catch (Exception ex) {
logger.error("Problem getting list of file. Skipping: {}", ex.getMessage(), ex);
}
return success;
}
private CandidateInfo candidate(Instant lastCandidateCreationTimestamp) {
EntityTransaction transaction = null;
EntityManager entityManager = null;
CandidateInfo candidate = null;
try (FakeRequestScope fakeRequestScope = new FakeRequestScope()) {
try {
QueryFactory queryFactory = this.applicationContext.getBean(QueryFactory.class);
EntityManagerFactory entityManagerFactory = this.applicationContext.getBean(EntityManagerFactory.class);
entityManager = entityManagerFactory.createEntityManager();
transaction = entityManager.getTransaction();
transaction.begin();
StorageFileQuery query = queryFactory.query(StorageFileQuery.class)
.canPurge(true)
.isPurged(false)
.createdAfter(lastCandidateCreationTimestamp);
query.setOrder(new Ordering().addAscending(StorageFile._createdAt));
StorageFileEntity item = query.first();
if (item != null) {
entityManager.flush();
candidate = new CandidateInfo();
candidate.setId(item.getId());
candidate.setCreatedAt(item.getCreatedAt());
}
transaction.commit();
} catch (OptimisticLockException ex) {
// we get this if/when someone else already modified the notifications. We want to essentially ignore this, and keep working
logger.debug("Concurrency exception getting file. Skipping: {} ", ex.getMessage());
if (transaction != null) transaction.rollback();
candidate = null;
} catch (Exception ex) {
logger.error("Problem getting list of file. Skipping: {}", ex.getMessage(), ex);
if (transaction != null) transaction.rollback();
candidate = null;
} finally {
if (entityManager != null) entityManager.close();
}
} catch (Exception ex) {
logger.error("Problem getting list of file. Skipping: {}", ex.getMessage(), ex);
}
return candidate;
}
}

View File

@ -0,0 +1,10 @@
package eu.eudat.service.storage;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties(StorageFileProperties.class)
public class StorageFileConfiguration {
}

View File

@ -0,0 +1,101 @@
package eu.eudat.service.storage;
import eu.eudat.commons.enums.StorageType;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.List;
@ConfigurationProperties(prefix = "storage.service")
public class StorageFileProperties {
private List<StorageConfig> storages;
private List<StaticFilesConfig> staticFiles;
private int tempStoreLifetimeSeconds;
public List<StorageConfig> getStorages() {
return storages;
}
public void setStorages(List<StorageConfig> storages) {
this.storages = storages;
}
public int getTempStoreLifetimeSeconds() {
return tempStoreLifetimeSeconds;
}
public void setTempStoreLifetimeSeconds(int tempStoreLifetimeSeconds) {
this.tempStoreLifetimeSeconds = tempStoreLifetimeSeconds;
}
public static class StorageConfig{
private StorageType type;
private String basePath;
public StorageType getType() {
return type;
}
public void setType(StorageType type) {
this.type = type;
}
public String getBasePath() {
return basePath;
}
public void setBasePath(String basePath) {
this.basePath = basePath;
}
}
public static class StaticFilesConfig{
private String externalUrls;
private String semantics;
private String h2020template;
private String h2020DatasetTemplate;
private String pidLinks;
public String getExternalUrls() {
return externalUrls;
}
public void setExternalUrls(String externalUrls) {
this.externalUrls = externalUrls;
}
public String getSemantics() {
return semantics;
}
public void setSemantics(String semantics) {
this.semantics = semantics;
}
public String getH2020template() {
return h2020template;
}
public void setH2020template(String h2020template) {
this.h2020template = h2020template;
}
public String getH2020DatasetTemplate() {
return h2020DatasetTemplate;
}
public void setH2020DatasetTemplate(String h2020DatasetTemplate) {
this.h2020DatasetTemplate = h2020DatasetTemplate;
}
public String getPidLinks() {
return pidLinks;
}
public void setPidLinks(String pidLinks) {
this.pidLinks = pidLinks;
}
}
}

View File

@ -0,0 +1,31 @@
package eu.eudat.service.storage;
import eu.eudat.commons.enums.StorageType;
import eu.eudat.model.StorageFile;
import eu.eudat.model.persist.StorageFilePersist;
import gr.cite.tools.fieldset.FieldSet;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationListener;
import java.io.IOException;
import java.nio.charset.Charset;
import java.time.Instant;
import java.util.UUID;
public interface StorageFileService extends ApplicationListener<ApplicationReadyEvent> {
StorageFile persistBytes(StorageFilePersist model, byte[] payload, FieldSet fields) throws IOException;
StorageFile persistString(StorageFilePersist model, String payload, FieldSet fields, Charset charset) throws IOException;
boolean moveToStorage(UUID fileId, StorageType storageType);
boolean copyToStorage(UUID fileId, StorageType storageType);
boolean exists(UUID fileId);
boolean fileRefExists(String fileRef, StorageType storageType);
void updatePurgeAt(UUID fileId, Instant purgeAt);
boolean purgeSafe(UUID fileId);
String readAsTextSafe(UUID fileId, Charset charset);
byte[] readAsBytesSafe(UUID fileId);
String readByFileRefAsTextSafe(String fileRef, StorageType storageType, Charset charset);
byte[] readByFileRefAsBytesSafe(String fileRef, StorageType storageType);
}

View File

@ -0,0 +1,281 @@
package eu.eudat.service.storage;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.commons.enums.StorageFilePermission;
import eu.eudat.commons.enums.StorageType;
import eu.eudat.commons.scope.user.UserScope;
import eu.eudat.data.StorageFileEntity;
import eu.eudat.model.Description;
import eu.eudat.model.StorageFile;
import eu.eudat.model.builder.StorageFileBuilder;
import eu.eudat.model.persist.StorageFilePersist;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.fieldset.FieldSet;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.stereotype.Service;
import org.springframework.util.FileCopyUtils;
import org.springframework.util.ResourceUtils;
import java.io.*;
import java.nio.charset.Charset;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.Locale;
import java.util.UUID;
@Service
public class StorageFileServiceImpl implements StorageFileService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(StorageFileServiceImpl.class));
private final EntityManager entityManager;
private final AuthorizationService authorizationService;
private final BuilderFactory builderFactory;
private final UserScope userScope;
private final StorageFileProperties config;
@Autowired
public StorageFileServiceImpl(
EntityManager entityManager,
AuthorizationService authorizationService,
BuilderFactory builderFactory,
UserScope userScope,
StorageFileProperties config
) {
this.entityManager = entityManager;
this.authorizationService = authorizationService;
this.builderFactory = builderFactory;
this.userScope = userScope;
this.config = config;
}
@Override
public void onApplicationEvent(ApplicationReadyEvent event) {
this.bootstrap();
}
private void bootstrap() {
if (this.config.getStorages() != null) {
for (StorageFileProperties.StorageConfig storage : this.config.getStorages()) {
Path path = Paths.get(storage.getBasePath());
if (!Files.exists(path)) {
try {
Files.createDirectories(path);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
}
}
@Override
public StorageFile persistBytes(StorageFilePersist model, byte[] payload, FieldSet fields) throws IOException {
StorageFileEntity storageFile = this.buildDataEntry(model);
File file = ResourceUtils.getFile(this.filePath(storageFile.getFileRef(), storageFile.getStorageType()));
if (!file.exists()) throw new FileAlreadyExistsException(storageFile.getName());
try (FileOutputStream fos = new FileOutputStream(file)) {
fos.write(payload);
}
this.entityManager.persist(storageFile);
this.entityManager.flush();
return this.builderFactory.builder(StorageFileBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(BaseFieldSet.build(fields, Description._id), storageFile);
}
@Override
public StorageFile persistString(StorageFilePersist model, String payload, FieldSet fields, Charset charset) throws IOException {
return this.persistBytes(model, payload.getBytes(charset), fields);
}
private StorageFileEntity buildDataEntry(StorageFilePersist model) {
StorageFileEntity data = new StorageFileEntity();
data.setFileRef(UUID.randomUUID().toString().replaceAll("-", "").toLowerCase(Locale.ROOT));
data.setName(model.getName());
data.setOwnerId(model.getOwnerId());
data.setExtension(model.getExtension());
data.setStorageType(model.getStorageType());
data.setMimeType(model.getMimeType());
data.setCreatedAt(Instant.now());
data.setPurgeAt(model.getLifetime() == null ? null : Instant.now().plus(model.getLifetime()));
return data;
}
@Override
public boolean moveToStorage(UUID fileId, StorageType storageType) {
try {
StorageFileEntity storageFile = this.entityManager.find(StorageFileEntity.class, fileId);
if (storageFile == null) return false;
this.authorizeForce(storageFile, StorageFilePermission.Read);
File file = ResourceUtils.getFile(this.filePath(storageFile.getFileRef(), storageFile.getStorageType()));
if (!file.exists()) return false;
File destinationFile = ResourceUtils.getFile(this.filePath(storageFile.getFileRef(), storageFile.getStorageType()));
boolean fileCopied = FileCopyUtils.copy(file, destinationFile) > 0;
if (!fileCopied) return false;
return file.delete();
}
catch (Exception ex) {
logger.warn("problem reading byte content of storage file " + fileId, ex);
return false;
}
}
@Override
public boolean copyToStorage(UUID fileId, StorageType storageType) {
try {
StorageFileEntity storageFile = this.entityManager.find(StorageFileEntity.class, fileId);
if (storageFile == null) return false;
this.authorizeForce(storageFile, StorageFilePermission.Read);
File file = ResourceUtils.getFile(this.filePath(storageFile.getFileRef(), storageFile.getStorageType()));
if (!file.exists()) return false;
File destinationFile = ResourceUtils.getFile(this.filePath(storageFile.getFileRef(), storageFile.getStorageType()));
return FileCopyUtils.copy(file, destinationFile) > 0;
}
catch (Exception ex) {
logger.warn("problem reading byte content of storage file " + fileId, ex);
return false;
}
}
@Override
public boolean exists(UUID fileId) {
try {
StorageFileEntity storageFile = this.entityManager.find(StorageFileEntity.class, fileId);
if (storageFile == null) return false;
this.authorizeForce(storageFile, StorageFilePermission.Read);
File file = ResourceUtils.getFile(this.filePath(storageFile.getFileRef(), storageFile.getStorageType()));
return file.exists();
}
catch (Exception ex) {
logger.warn("problem reading byte content of storage file " + fileId, ex);
return false;
}
}
@Override
public boolean fileRefExists(String fileRef, StorageType storageType) {
File file = null;
try {
file = ResourceUtils.getFile(this.filePath(fileRef, storageType));
return file.exists();
} catch (Exception ex) {
logger.warn("problem reading byte content of storage file " + fileRef, ex);
return false;
}
}
@Override
public void updatePurgeAt(UUID fileId, Instant purgeAt) {
StorageFileEntity storageFile = this.entityManager.find(StorageFileEntity.class, fileId);
if (storageFile == null) return;
storageFile.setPurgeAt(purgeAt);
this.entityManager.merge(storageFile);
this.entityManager.flush();
}
@Override
public boolean purgeSafe(UUID fileId) {
try {
StorageFileEntity storageFile = this.entityManager.find(StorageFileEntity.class, fileId);
if (storageFile == null) return false;
storageFile.setPurgedAt(Instant.now());
this.entityManager.merge(storageFile);
this.entityManager.flush();
File file = ResourceUtils.getFile(this.filePath(storageFile.getFileRef(), storageFile.getStorageType()));
if (!file.exists()) return true;
return file.delete();
}
catch (Exception ex) {
logger.warn("problem reading byte content of storage file " + fileId, ex);
return false;
}
}
@Override
public String readAsTextSafe(UUID fileId, Charset charset) {
byte[] bytes = this.readAsBytesSafe(fileId);
return bytes == null ? null : new String(bytes, charset);
}
@Override
public byte[] readAsBytesSafe(UUID fileId) {
byte[] bytes = null;
try {
StorageFileEntity storageFile = this.entityManager.find(StorageFileEntity.class, fileId);
if (storageFile == null) return null;
this.authorizeForce(storageFile, StorageFilePermission.Read);
bytes = this.readByFileRefAsBytesSafe(storageFile.getFileRef(), storageFile.getStorageType());
}
catch (Exception ex) {
logger.warn("problem reading byte content of storage file " + fileId, ex);
}
return bytes;
}
@Override
public String readByFileRefAsTextSafe(String fileRef, StorageType storageType, Charset charset) {
byte[] bytes = this.readByFileRefAsBytesSafe(fileRef, storageType);
return bytes == null ? null : new String(bytes, charset);
}
@Override
public byte[] readByFileRefAsBytesSafe(String fileRef, StorageType storageType) {
byte[] bytes = null;
try {
File file = ResourceUtils.getFile(this.filePath(fileRef, storageType));
if (!file.exists()) return null;
try(InputStream inputStream = new FileInputStream(file)){
bytes = inputStream.readAllBytes();
};
}
catch (Exception ex) {
logger.warn("problem reading byte content of storage file " + fileRef, ex);
}
return bytes;
}
private String filePath(String fileRef, StorageType storageType)
{
StorageFileProperties.StorageConfig storageFileConfig = this.config.getStorages().stream().filter(x -> storageType.equals(x.getType())).findFirst().orElse(null);
if (storageFileConfig == null) throw new MyApplicationException("Storage " + storageType + " not found");
return Paths.get(storageFileConfig.getBasePath(), fileRef).toString();
}
private void authorizeForce(StorageFileEntity storageFile, StorageFilePermission storageFilePermission) {
if (storageFile.getOwnerId() != null && storageFile.getOwnerId().equals(this.userScope.getUserIdSafe())){
return;
}
this.authorizationService.authorizeForce(Permission.BrowseStorageFile);
}
}