From c5461dbc6205cafc77fbbba3cb1dba1177f8d22f Mon Sep 17 00:00:00 2001 From: sgiannopoulos Date: Tue, 28 Nov 2023 15:05:58 +0200 Subject: [PATCH] add storage service --- .../java/eu/eudat/audit/AuditableAction.java | 3 + .../eu/eudat/authorization/Permission.java | 7 + .../commons/enums/StorageFilePermission.java | 30 +++ .../eu/eudat/commons/enums/StorageType.java | 30 +++ .../commons/fake/FakeRequestAttributes.java | 126 +++++++++++ .../eudat/commons/fake/FakeRequestScope.java | 44 ++++ .../java/eu/eudat/data/StorageFileEntity.java | 141 ++++++++++++ .../enums/StorageTypeConverter.java | 12 + .../main/java/eu/eudat/model/StorageFile.java | 130 +++++++++++ .../model/builder/StorageFileBuilder.java | 111 ++++++++++ .../model/censorship/StorageFileCensor.java | 45 ++++ .../model/persist/StorageFilePersist.java | 86 ++++++++ .../java/eu/eudat/query/StorageFileQuery.java | 206 ++++++++++++++++++ .../controllers/v2/StorageFileController.java | 142 ++++++++++++ .../src/main/resources/config/application.yml | 1 + .../src/main/resources/config/permissions.yml | 21 ++ .../main/resources/config/storage-devel.yml | 7 + .../web/src/main/resources/config/storage.yml | 6 + .../updates/00.01.024_addStorageFile.sql | 32 +++ 19 files changed, 1180 insertions(+) create mode 100644 dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageFilePermission.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageType.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/commons/fake/FakeRequestAttributes.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/commons/fake/FakeRequestScope.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/data/StorageFileEntity.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/data/converters/enums/StorageTypeConverter.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/model/StorageFile.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/model/builder/StorageFileBuilder.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/model/censorship/StorageFileCensor.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/model/persist/StorageFilePersist.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/query/StorageFileQuery.java create mode 100644 dmp-backend/web/src/main/java/eu/eudat/controllers/v2/StorageFileController.java create mode 100644 dmp-backend/web/src/main/resources/config/storage-devel.yml create mode 100644 dmp-backend/web/src/main/resources/config/storage.yml create mode 100644 dmp-db-scema/updates/00.01.024_addStorageFile.sql diff --git a/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java b/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java index 2ec15c6f1..8bef6d46b 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java +++ b/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java @@ -79,6 +79,9 @@ public class AuditableAction { public static final EventId Tenant_Lookup = new EventId(12001, "Tenant_Lookup"); public static final EventId Tenant_Persist = new EventId(12002, "Tenant_Persist"); public static final EventId Tenant_Delete = new EventId(12003, "Tenant_Delete"); + + public static final EventId StorageFile_Download = new EventId(13000, "StorageFile_Download"); + public static final EventId StorageFile_Upload = new EventId(13001, "StorageFile_Upload"); } diff --git a/dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java b/dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java index 82d82a3a3..b9390c802 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java +++ b/dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java @@ -49,6 +49,13 @@ public final class Permission { public static String EditUser = "EditUser"; public static String DeleteUser = "DeleteUser"; public static String ExportUsers = "ExportUsers"; + + + + //StorageFile + public static String BrowseStorageFile = "BrowseStorageFile"; + public static String EditStorageFile = "EditStorageFile"; + public static String DeleteStorageFile = "DeleteStorageFile"; //DescriptionTemplateType public static String BrowseDescriptionTemplateType = "BrowseDescriptionTemplateType"; diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageFilePermission.java b/dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageFilePermission.java new file mode 100644 index 000000000..b62afc910 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageFilePermission.java @@ -0,0 +1,30 @@ +package eu.eudat.commons.enums; + +import com.fasterxml.jackson.annotation.JsonValue; +import eu.eudat.data.converters.enums.DatabaseEnum; + +import java.util.Map; + +public enum StorageFilePermission implements DatabaseEnum { + + Read((short) 0), + Write((short) 1); + + private final Short value; + + StorageFilePermission(Short value) { + this.value = value; + } + + @JsonValue + public Short getValue() { + return value; + } + + private static final Map map = EnumUtils.getEnumValueMap(StorageFilePermission.class); + + public static StorageFilePermission of(Short i) { + return map.get(i); + } + +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageType.java b/dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageType.java new file mode 100644 index 000000000..59e0b1bae --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageType.java @@ -0,0 +1,30 @@ +package eu.eudat.commons.enums; + +import com.fasterxml.jackson.annotation.JsonValue; +import eu.eudat.data.converters.enums.DatabaseEnum; + +import java.util.Map; + +public enum StorageType implements DatabaseEnum { + + Temp((short) 0), + Main((short) 1); + + private final Short value; + + StorageType(Short value) { + this.value = value; + } + + @JsonValue + public Short getValue() { + return value; + } + + private static final Map map = EnumUtils.getEnumValueMap(StorageType.class); + + public static StorageType of(Short i) { + return map.get(i); + } + +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/fake/FakeRequestAttributes.java b/dmp-backend/core/src/main/java/eu/eudat/commons/fake/FakeRequestAttributes.java new file mode 100644 index 000000000..021c1924b --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/fake/FakeRequestAttributes.java @@ -0,0 +1,126 @@ +package eu.eudat.commons.fake; + +import org.jetbrains.annotations.NotNull; +import org.springframework.util.Assert; +import org.springframework.web.context.request.RequestAttributes; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +public class FakeRequestAttributes implements RequestAttributes { + private final Map requestAttributeMap = new HashMap<>(); + private final Map requestDestructionCallbacks = new LinkedHashMap<>(8); + private volatile boolean requestActive = true; + + @Override + public Object getAttribute(@NotNull String name, int scope) { + if (scope == RequestAttributes.SCOPE_REQUEST) { + if (!isRequestActive()) { + throw new IllegalStateException("Cannot ask for request attribute - request is not active anymore!"); + } + return this.requestAttributeMap.get(name); + } else { + throw new IllegalStateException("Only " + RequestAttributes.SCOPE_REQUEST + " allowed for " + FakeRequestAttributes.class.getSimpleName()); + } + } + + @Override + public void setAttribute(@NotNull String name, @NotNull Object value, int scope) { + if (scope == RequestAttributes.SCOPE_REQUEST) { + if (!isRequestActive()) { + throw new IllegalStateException("Cannot set request attribute - request is not active anymore!"); + } + this.requestAttributeMap.put(name, value); + } else { + throw new IllegalStateException("Only " + RequestAttributes.SCOPE_REQUEST + " allowed for " + FakeRequestAttributes.class.getSimpleName()); + } + } + + @Override + public void removeAttribute(@NotNull String name, int scope) { + if (scope == RequestAttributes.SCOPE_REQUEST) { + if (isRequestActive()) { + removeRequestDestructionCallback(name); + this.requestAttributeMap.remove(name); + } + } else { + throw new IllegalStateException("Only " + RequestAttributes.SCOPE_REQUEST + " allowed for " + FakeRequestAttributes.class.getSimpleName()); + } + } + + @Override + public String @NotNull [] getAttributeNames(int scope) { + if (scope == RequestAttributes.SCOPE_REQUEST) { + if (!isRequestActive()) { + throw new IllegalStateException("Cannot ask for request attributes - request is not active anymore!"); + } + return this.requestAttributeMap.keySet().toArray(new String[0]); + } else { + throw new IllegalStateException("Only " + RequestAttributes.SCOPE_REQUEST + " allowed for " + FakeRequestAttributes.class.getSimpleName()); + } + //return new String[0]; + } + + @Override + public void registerDestructionCallback(@NotNull String name, @NotNull Runnable callback, int scope) { + if (scope == SCOPE_REQUEST) { + registerRequestDestructionCallback(name, callback); + } else { + throw new IllegalStateException("Only " + RequestAttributes.SCOPE_REQUEST + " allowed for " + FakeRequestAttributes.class.getSimpleName()); + } + } + + protected final void registerRequestDestructionCallback(String name, Runnable callback) { + Assert.notNull(name, "Name must not be null"); + Assert.notNull(callback, "Callback must not be null"); + synchronized (this.requestDestructionCallbacks) { + this.requestDestructionCallbacks.put(name, callback); + } + } + + @Override + public Object resolveReference(@NotNull String key) { + // Not supported + return null; + } + + @Override + public @NotNull String getSessionId() { + return ""; + } + + @Override + public @NotNull Object getSessionMutex() { + return new Object(); + } + + public void requestCompleted() { + executeRequestDestructionCallbacks(); + for (String name : getAttributeNames(RequestAttributes.SCOPE_REQUEST)) { + this.removeAttribute(name, RequestAttributes.SCOPE_REQUEST); + } + this.requestActive = false; + } + + private boolean isRequestActive() { + return this.requestActive; + } + + + private void removeRequestDestructionCallback(String name) { + Assert.notNull(name, "Name must not be null"); + synchronized (this.requestDestructionCallbacks) { + this.requestDestructionCallbacks.remove(name); + } + } + + private void executeRequestDestructionCallbacks() { + synchronized (this.requestDestructionCallbacks) { + for (Runnable runnable : this.requestDestructionCallbacks.values()) { + runnable.run(); + } + this.requestDestructionCallbacks.clear(); + } + } +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/fake/FakeRequestScope.java b/dmp-backend/core/src/main/java/eu/eudat/commons/fake/FakeRequestScope.java new file mode 100644 index 000000000..3fb372149 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/fake/FakeRequestScope.java @@ -0,0 +1,44 @@ +package eu.eudat.commons.fake; + +import org.springframework.web.context.request.RequestAttributes; +import org.springframework.web.context.request.RequestContextHolder; + +import java.io.Closeable; + +public class FakeRequestScope implements Closeable { + private RequestAttributes initialRequestAttributes = null; + private FakeRequestAttributes currentRequestAttributes = null; + boolean isInUse = false; + + public FakeRequestScope() { + this.reset(); + } + + public final void reset() { + this.close(); + this.isInUse = true; + + this.initialRequestAttributes = RequestContextHolder.getRequestAttributes(); + this.currentRequestAttributes = new FakeRequestAttributes(); + RequestContextHolder.setRequestAttributes(this.currentRequestAttributes); + } + + @Override + public void close() { + if (!this.isInUse) + return; + this.isInUse = false; + + if (initialRequestAttributes != null) + RequestContextHolder.setRequestAttributes(initialRequestAttributes); + else + RequestContextHolder.resetRequestAttributes(); + + if (currentRequestAttributes != null) + currentRequestAttributes.requestCompleted(); + + this.initialRequestAttributes = null; + this.currentRequestAttributes = null; + } + +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/data/StorageFileEntity.java b/dmp-backend/core/src/main/java/eu/eudat/data/StorageFileEntity.java new file mode 100644 index 000000000..e42150fef --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/data/StorageFileEntity.java @@ -0,0 +1,141 @@ +package eu.eudat.data; + + +import eu.eudat.commons.enums.StorageType; +import eu.eudat.data.converters.enums.StorageTypeConverter; +import jakarta.persistence.*; + +import java.time.Instant; +import java.util.UUID; + +@Entity +@Table(name = "\"StorageFile\"") +public class StorageFileEntity { + + @Id + @Column(name = "id", columnDefinition = "uuid", updatable = false, nullable = false) + private UUID id; + public final static String _id = "id"; + + @Column(name = "file_ref", length = _fileRefLen, nullable = false) + private String fileRef; + public final static String _fileRef = "fileRef"; + public final static int _fileRefLen = 100; + + @Column(name = "name", length = _nameLen, nullable = false) + private String name; + public final static String _name = "name"; + public final static int _nameLen = 250; + + @Column(name = "extension", length = _extensionLen, nullable = false) + private String extension; + public final static String _extension = "extension"; + public final static int _extensionLen = 10; + + @Column(name = "mime_type", length = _mimeTypeLen, nullable = false) + private String mimeType; + public final static String _mimeType = "mimeType"; + public final static int _mimeTypeLen = 200; + + @Column(name = "storage_type", nullable = false) + @Convert(converter = StorageTypeConverter.class) + private StorageType storageType; + public final static String _storageType = "storageType"; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + public final static String _createdAt = "createdAt"; + + @Column(name = "purge_at", nullable = true) + private Instant purgeAt; + public final static String _purgeAt = "purgeAt"; + + @Column(name = "purged_at", nullable = true) + private Instant purgedAt; + public final static String _purgedAt = "purgedAt"; + + @Column(name = "owner", nullable = true) + private UUID ownerId; + public final static String _ownerId = "ownerId"; + + public UUID getId() { + return id; + } + + public void setId(UUID id) { + this.id = id; + } + + public String getFileRef() { + return fileRef; + } + + public void setFileRef(String fileRef) { + this.fileRef = fileRef; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getExtension() { + return extension; + } + + public void setExtension(String extension) { + this.extension = extension; + } + + public String getMimeType() { + return mimeType; + } + + public void setMimeType(String mimeType) { + this.mimeType = mimeType; + } + + public StorageType getStorageType() { + return storageType; + } + + public void setStorageType(StorageType storageType) { + this.storageType = storageType; + } + + + public Instant getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(Instant createdAt) { + this.createdAt = createdAt; + } + + public Instant getPurgeAt() { + return purgeAt; + } + + public void setPurgeAt(Instant purgeAt) { + this.purgeAt = purgeAt; + } + + public Instant getPurgedAt() { + return purgedAt; + } + + public void setPurgedAt(Instant purgedAt) { + this.purgedAt = purgedAt; + } + + public UUID getOwnerId() { + return ownerId; + } + + public void setOwnerId(UUID ownerId) { + this.ownerId = ownerId; + } +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/data/converters/enums/StorageTypeConverter.java b/dmp-backend/core/src/main/java/eu/eudat/data/converters/enums/StorageTypeConverter.java new file mode 100644 index 000000000..7baa726a0 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/data/converters/enums/StorageTypeConverter.java @@ -0,0 +1,12 @@ +package eu.eudat.data.converters.enums; + +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.StorageType; +import jakarta.persistence.Converter; + +@Converter +public class StorageTypeConverter extends DatabaseEnumConverter { + public StorageType of(Short i) { + return StorageType.of(i); + } +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/StorageFile.java b/dmp-backend/core/src/main/java/eu/eudat/model/StorageFile.java new file mode 100644 index 000000000..763b3a1d5 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/model/StorageFile.java @@ -0,0 +1,130 @@ +package eu.eudat.model; + +import eu.eudat.commons.enums.StorageType; + +import java.time.Instant; +import java.util.UUID; + +public class StorageFile { + + private UUID id; + public final static String _id = "id"; + + private String fileRef; + public final static String _fileRef = "fileRef"; + + private String name; + public final static String _name = "name"; + + private String fullName; + public final static String _fullName = "fullName"; + + private String extension; + public final static String _extension = "extension"; + + private String mimeType; + public final static String _mimeType = "mimeType"; + + private StorageType storageType; + public final static String _storageType = "storageType"; + + private Instant createdAt; + public final static String _createdAt = "createdAt"; + + private Instant purgeAt; + public final static String _purgeAt = "purgeAt"; + + private Instant purgedAt; + public final static String _purgedAt = "purgedAt"; + + private User owner; + public final static String _owner = "owner"; + + public UUID getId() { + return id; + } + + public void setId(UUID id) { + this.id = id; + } + + public String getFileRef() { + return fileRef; + } + + public void setFileRef(String fileRef) { + this.fileRef = fileRef; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFullName() { + return fullName; + } + + public void setFullName(String fullName) { + this.fullName = fullName; + } + + public String getExtension() { + return extension; + } + + public void setExtension(String extension) { + this.extension = extension; + } + + public String getMimeType() { + return mimeType; + } + + public void setMimeType(String mimeType) { + this.mimeType = mimeType; + } + + public StorageType getStorageType() { + return storageType; + } + + public void setStorageType(StorageType storageType) { + this.storageType = storageType; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(Instant createdAt) { + this.createdAt = createdAt; + } + + public Instant getPurgeAt() { + return purgeAt; + } + + public void setPurgeAt(Instant purgeAt) { + this.purgeAt = purgeAt; + } + + public Instant getPurgedAt() { + return purgedAt; + } + + public void setPurgedAt(Instant purgedAt) { + this.purgedAt = purgedAt; + } + + public User getOwner() { + return owner; + } + + public void setOwner(User owner) { + this.owner = owner; + } +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/builder/StorageFileBuilder.java b/dmp-backend/core/src/main/java/eu/eudat/model/builder/StorageFileBuilder.java new file mode 100644 index 000000000..08cb7411e --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/model/builder/StorageFileBuilder.java @@ -0,0 +1,111 @@ +package eu.eudat.model.builder; + +import eu.eudat.authorization.AuthorizationFlags; +import eu.eudat.convention.ConventionService; +import eu.eudat.data.StorageFileEntity; +import eu.eudat.model.*; +import eu.eudat.query.*; +import gr.cite.tools.data.builder.BuilderFactory; +import gr.cite.tools.data.query.QueryFactory; +import gr.cite.tools.exception.MyApplicationException; +import gr.cite.tools.fieldset.BaseFieldSet; +import gr.cite.tools.fieldset.FieldSet; +import gr.cite.tools.logging.DataLogEntry; +import gr.cite.tools.logging.LoggerService; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import java.util.*; +import java.util.stream.Collectors; + +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class StorageFileBuilder extends BaseBuilder { + + private final QueryFactory queryFactory; + + private final BuilderFactory builderFactory; + + private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); + + @Autowired + public StorageFileBuilder( + ConventionService conventionService, + QueryFactory queryFactory, + BuilderFactory builderFactory) { + super(conventionService, new LoggerService(LoggerFactory.getLogger(StorageFileBuilder.class))); + this.queryFactory = queryFactory; + this.builderFactory = builderFactory; + } + + public StorageFileBuilder authorize(EnumSet values) { + this.authorize = values; + return this; + } + + @Override + public List build(FieldSet fields, List data) throws MyApplicationException { + this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0)); + this.logger.trace(new DataLogEntry("requested fields", fields)); + if (fields == null || data == null || fields.isEmpty()) + return new ArrayList<>(); + + FieldSet userFields = fields.extractPrefixed(this.asPrefix(StorageFile._owner)); + Map userItemsMap = this.collectUsers(userFields, data); + + List models = new ArrayList<>(); + for (StorageFileEntity d : data) { + StorageFile m = new StorageFile(); + if (fields.hasField(this.asIndexer(StorageFile._id))) m.setId(d.getId()); + if (fields.hasField(this.asIndexer(StorageFile._name))) m.setName(d.getName()); + if (fields.hasField(this.asIndexer(StorageFile._fileRef))) m.setFileRef(d.getFileRef()); + if (fields.hasField(this.asIndexer(StorageFile._extension))) m.setExtension(d.getExtension()); + if (fields.hasField(this.asIndexer(StorageFile._createdAt))) m.setCreatedAt(d.getCreatedAt()); + if (fields.hasField(this.asIndexer(StorageFile._mimeType))) m.setMimeType(d.getMimeType()); + if (fields.hasField(this.asIndexer(StorageFile._storageType))) m.setStorageType(d.getStorageType()); + if (fields.hasField(this.asIndexer(StorageFile._purgeAt))) m.setPurgeAt(d.getPurgeAt()); + if (fields.hasField(this.asIndexer(StorageFile._purgedAt))) m.setPurgedAt(d.getPurgedAt()); + if (fields.hasField(this.asIndexer(StorageFile._fullName))) m.setFullName(d.getName() + (d.getExtension().startsWith(".") ? "" : ".") + d.getExtension()); + if (!userFields.isEmpty() && userItemsMap != null && userItemsMap.containsKey(d.getOwnerId())) m.setOwner(userItemsMap.get(d.getOwnerId())); + models.add(m); + } + + this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0)); + + return models; + } + + private Map collectUsers(FieldSet fields, List data) throws MyApplicationException { + if (fields.isEmpty() || data.isEmpty()) + return null; + this.logger.debug("checking related - {}", User.class.getSimpleName()); + + Map itemMap; + if (!fields.hasOtherField(this.asIndexer(User._id))) { + itemMap = this.asEmpty( + data.stream().map(StorageFileEntity::getOwnerId).distinct().collect(Collectors.toList()), + x -> { + User item = new User(); + item.setId(x); + return item; + }, + User::getId); + } else { + FieldSet clone = new BaseFieldSet(fields.getFields()).ensure(User._id); + UserQuery q = this.queryFactory.query(UserQuery.class).authorize(this.authorize).ids(data.stream().map(StorageFileEntity::getOwnerId).distinct().collect(Collectors.toList())); + itemMap = this.builderFactory.builder(UserBuilder.class).authorize(this.authorize).asForeignKey(q, clone, User::getId); + } + if (!fields.hasField(User._id)) { + itemMap.forEach((id, item) -> { + if (item != null) + item.setId(null); + }); + } + + return itemMap; + } + +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/censorship/StorageFileCensor.java b/dmp-backend/core/src/main/java/eu/eudat/model/censorship/StorageFileCensor.java new file mode 100644 index 000000000..e72a54422 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/model/censorship/StorageFileCensor.java @@ -0,0 +1,45 @@ +package eu.eudat.model.censorship; + +import eu.eudat.authorization.Permission; +import eu.eudat.convention.ConventionService; +import eu.eudat.model.StorageFile; +import gr.cite.commons.web.authz.service.AuthorizationService; +import gr.cite.tools.data.censor.CensorFactory; +import gr.cite.tools.fieldset.FieldSet; +import gr.cite.tools.logging.DataLogEntry; +import gr.cite.tools.logging.LoggerService; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import java.util.UUID; + +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class StorageFileCensor extends BaseCensor { + + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(StorageFileCensor.class)); + + protected final AuthorizationService authService; + + protected final CensorFactory censorFactory; + + public StorageFileCensor(ConventionService conventionService, AuthorizationService authService, CensorFactory censorFactory) { + super(conventionService); + this.authService = authService; + this.censorFactory = censorFactory; + } + + public void censor(FieldSet fields, UUID userId) { + logger.debug(new DataLogEntry("censoring fields", fields)); + if (fields == null || fields.isEmpty()) + return; + + this.authService.authorizeForce(Permission.BrowseStorageFile); + + FieldSet ownerFields = fields.extractPrefixed(this.asIndexerPrefix(StorageFile._owner)); + this.censorFactory.censor(UserCensor.class).censor(ownerFields, userId); + } + +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/persist/StorageFilePersist.java b/dmp-backend/core/src/main/java/eu/eudat/model/persist/StorageFilePersist.java new file mode 100644 index 000000000..55a8df9d2 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/model/persist/StorageFilePersist.java @@ -0,0 +1,86 @@ +package eu.eudat.model.persist; + +import eu.eudat.commons.enums.StorageType; +import eu.eudat.commons.validation.ValidEnum; +import eu.eudat.data.StorageFileEntity; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; + +import java.time.Duration; +import java.util.UUID; + +public class StorageFilePersist { + + @NotNull(message = "{validation.empty}") + @NotEmpty(message = "{validation.empty}") + @Size(max = StorageFileEntity._nameLen, message = "{validation.largerthanmax}") + private String name; + + @NotNull(message = "{validation.empty}") + @NotEmpty(message = "{validation.empty}") + @Size(max = StorageFileEntity._extensionLen, message = "{validation.largerthanmax}") + private String extension; + + + + @NotNull(message = "{validation.empty}") + @NotEmpty(message = "{validation.empty}") + @Size(max = StorageFileEntity._mimeTypeLen, message = "{validation.largerthanmax}") + private String mimeType; + + @NotNull(message = "{validation.empty}") + @ValidEnum(message = "{validation.empty}") + private StorageType storageType; + private Duration lifetime; + + private UUID ownerId; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getExtension() { + return extension; + } + + public void setExtension(String extension) { + this.extension = extension; + } + + public String getMimeType() { + return mimeType; + } + + public void setMimeType(String mimeType) { + this.mimeType = mimeType; + } + + public StorageType getStorageType() { + return storageType; + } + + public void setStorageType(StorageType storageType) { + this.storageType = storageType; + } + + public UUID getOwnerId() { + return ownerId; + } + + public void setOwnerId(UUID ownerId) { + this.ownerId = ownerId; + } + + public Duration getLifetime() { + return lifetime; + } + + public void setLifetime(Duration lifetime) { + this.lifetime = lifetime; + } +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/query/StorageFileQuery.java b/dmp-backend/core/src/main/java/eu/eudat/query/StorageFileQuery.java new file mode 100644 index 000000000..2aef9ced3 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/query/StorageFileQuery.java @@ -0,0 +1,206 @@ +package eu.eudat.query; + +import eu.eudat.authorization.AuthorizationFlags; +import eu.eudat.authorization.Permission; +import eu.eudat.commons.enums.StorageType; +import eu.eudat.commons.scope.user.UserScope; +import eu.eudat.data.StorageFileEntity; +import eu.eudat.model.StorageFile; +import gr.cite.commons.web.authz.service.AuthorizationService; +import gr.cite.tools.data.query.FieldResolver; +import gr.cite.tools.data.query.QueryBase; +import gr.cite.tools.data.query.QueryContext; +import jakarta.persistence.Tuple; +import jakarta.persistence.criteria.CriteriaBuilder; +import jakarta.persistence.criteria.Predicate; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import java.time.Instant; +import java.util.*; + +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class StorageFileQuery extends QueryBase { + private String like; + private Collection ids; + private Boolean canPurge; + private Boolean isPurged; + private Instant createdAfter; + private Collection excludedIds; + private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); + + private final UserScope userScope; + private final AuthorizationService authService; + public StorageFileQuery(UserScope userScope, AuthorizationService authService) { + this.userScope = userScope; + this.authService = authService; + } + + public StorageFileQuery like(String value) { + this.like = value; + return this; + } + + public StorageFileQuery ids(UUID value) { + this.ids = List.of(value); + return this; + } + + public StorageFileQuery ids(UUID... value) { + this.ids = Arrays.asList(value); + return this; + } + + public StorageFileQuery ids(Collection values) { + this.ids = values; + return this; + } + + public StorageFileQuery createdAfter(Instant value) { + this.createdAfter = value; + return this; + } + + public StorageFileQuery canPurge(Boolean value) { + this.canPurge = value; + return this; + } + + public StorageFileQuery isPurged(Boolean value) { + this.isPurged = value; + return this; + } + + public StorageFileQuery excludedIds(Collection values) { + this.excludedIds = values; + return this; + } + + public StorageFileQuery excludedIds(UUID value) { + this.excludedIds = List.of(value); + return this; + } + + public StorageFileQuery excludedIds(UUID... value) { + this.excludedIds = Arrays.asList(value); + return this; + } + + public StorageFileQuery authorize(EnumSet values) { + this.authorize = values; + return this; + } + + @Override + protected Boolean isFalseQuery() { + return + this.isEmpty(this.ids) || + this.isEmpty(this.excludedIds) ; + } + + @Override + protected Class entityClass() { + return StorageFileEntity.class; + } + + @Override + protected Predicate applyAuthZ(QueryContext queryContext) { + if (this.authorize.contains(AuthorizationFlags.None)) return null; + if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseStorageFile)) return null; + UUID userId; + if (this.authorize.contains(AuthorizationFlags.Owner)) userId = this.userScope.getUserIdSafe(); + else userId = null; + + List predicates = new ArrayList<>(); + if (userId != null) { + predicates.add(queryContext.CriteriaBuilder.in(queryContext.Root.get(StorageFileEntity._ownerId)).value(userId)); + } + if (!predicates.isEmpty()) { + Predicate[] predicatesArray = predicates.toArray(new Predicate[0]); + return queryContext.CriteriaBuilder.and(predicatesArray); + } else { + return queryContext.CriteriaBuilder.or(); //Creates a false query + } + } + + @Override + protected Predicate applyFilters(QueryContext queryContext) { + List predicates = new ArrayList<>(); + if (this.like != null && !this.like.isEmpty()) { + predicates.add( queryContext.CriteriaBuilder.like(queryContext.Root.get(StorageFileEntity._name), this.like)); + } + if (this.ids != null) { + CriteriaBuilder.In inClause = queryContext.CriteriaBuilder.in(queryContext.Root.get(StorageFileEntity._id)); + for (UUID item : this.ids) + inClause.value(item); + predicates.add(inClause); + } + if (this.createdAfter != null) { + predicates.add(queryContext.CriteriaBuilder.greaterThan(queryContext.Root.get(StorageFileEntity._createdAt), this.createdAfter)); + } + if (this.canPurge != null) { + predicates.add( + queryContext.CriteriaBuilder.and( + queryContext.CriteriaBuilder.isNull(queryContext.Root.get(StorageFileEntity._purgeAt)).not(), + queryContext.CriteriaBuilder.lessThan(queryContext.Root.get(StorageFileEntity._purgeAt), Instant.now()) + ) + ); + } + if (this.isPurged != null) { + if (!this.isPurged){ + predicates.add(queryContext.CriteriaBuilder.isNull(queryContext.Root.get(StorageFileEntity._purgedAt)).not()); + } else { + predicates.add(queryContext.CriteriaBuilder.isNull(queryContext.Root.get(StorageFileEntity._purgedAt))); + } + } + if (this.excludedIds != null) { + CriteriaBuilder.In notInClause = queryContext.CriteriaBuilder.in(queryContext.Root.get(StorageFileEntity._id)); + for (UUID item : this.excludedIds) + notInClause.value(item); + predicates.add(notInClause.not()); + } + + if (!predicates.isEmpty()) { + Predicate[] predicatesArray = predicates.toArray(new Predicate[0]); + return queryContext.CriteriaBuilder.and(predicatesArray); + } else { + return null; + } + } + + @Override + protected String fieldNameOf(FieldResolver item) { + if (item.match(StorageFile._id)) return StorageFileEntity._id; + else if (item.match(StorageFile._name)) return StorageFileEntity._name; + else if (item.match(StorageFile._fileRef)) return StorageFileEntity._fileRef; + else if (item.match(StorageFile._fullName)) return StorageFileEntity._name; + else if (item.match(StorageFile._extension)) return StorageFileEntity._extension; + else if (item.match(StorageFile._mimeType)) return StorageFileEntity._mimeType; + else if (item.match(StorageFile._storageType)) return StorageFileEntity._storageType; + else if (item.match(StorageFile._createdAt)) return StorageFileEntity._createdAt; + else if (item.match(StorageFile._purgeAt)) return StorageFileEntity._purgeAt; + else if (item.match(StorageFile._purgedAt)) return StorageFileEntity._purgedAt; + else if (item.match(StorageFile._owner)) return StorageFileEntity._ownerId; + else if (item.prefix(StorageFile._owner)) return StorageFileEntity._ownerId; + else return null; + } + + @Override + protected StorageFileEntity convert(Tuple tuple, Set columns) { + StorageFileEntity item = new StorageFileEntity(); + item.setId(QueryBase.convertSafe(tuple, columns, StorageFileEntity._id, UUID.class)); + item.setName(QueryBase.convertSafe(tuple, columns, StorageFileEntity._name, String.class)); + item.setFileRef(QueryBase.convertSafe(tuple, columns, StorageFileEntity._fileRef, String.class)); + item.setExtension(QueryBase.convertSafe(tuple, columns, StorageFileEntity._extension, String.class)); + item.setMimeType(QueryBase.convertSafe(tuple, columns, StorageFileEntity._mimeType, String.class)); + item.setStorageType(QueryBase.convertSafe(tuple, columns, StorageFileEntity._storageType, StorageType.class)); + item.setCreatedAt(QueryBase.convertSafe(tuple, columns, StorageFileEntity._createdAt, Instant.class)); + item.setPurgeAt(QueryBase.convertSafe(tuple, columns, StorageFileEntity._purgeAt, Instant.class)); + item.setPurgedAt(QueryBase.convertSafe(tuple, columns, StorageFileEntity._purgedAt, Instant.class)); + item.setOwnerId(QueryBase.convertSafe(tuple, columns, StorageFileEntity._ownerId, UUID.class)); + return item; + } + +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/StorageFileController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/StorageFileController.java new file mode 100644 index 000000000..6b228dfb1 --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/StorageFileController.java @@ -0,0 +1,142 @@ +package eu.eudat.controllers.v2; + +import com.fasterxml.jackson.core.JsonProcessingException; +import eu.eudat.audit.AuditableAction; +import eu.eudat.authorization.AuthorizationFlags; +import eu.eudat.authorization.Permission; +import eu.eudat.commons.enums.*; +import eu.eudat.commons.scope.user.UserScope; +import eu.eudat.convention.ConventionService; +import eu.eudat.data.StorageFileEntity; +import eu.eudat.model.*; +import eu.eudat.model.builder.DescriptionBuilder; +import eu.eudat.model.censorship.DescriptionCensor; +import eu.eudat.model.censorship.PublicDescriptionCensor; +import eu.eudat.model.persist.DescriptionPersist; +import eu.eudat.model.persist.StorageFilePersist; +import eu.eudat.model.result.QueryResult; +import eu.eudat.query.DescriptionQuery; +import eu.eudat.query.DmpQuery; +import eu.eudat.query.StorageFileQuery; +import eu.eudat.query.lookup.DescriptionLookup; +import eu.eudat.service.description.DescriptionService; +import eu.eudat.service.elastic.ElasticQueryHelperService; +import eu.eudat.service.storage.StorageFileProperties; +import eu.eudat.service.storage.StorageFileService; +import gr.cite.commons.web.authz.service.AuthorizationService; +import gr.cite.tools.auditing.AuditService; +import gr.cite.tools.data.builder.BuilderFactory; +import gr.cite.tools.data.censor.CensorFactory; +import gr.cite.tools.data.query.QueryFactory; +import gr.cite.tools.exception.MyApplicationException; +import gr.cite.tools.exception.MyForbiddenException; +import gr.cite.tools.exception.MyNotFoundException; +import gr.cite.tools.fieldset.BaseFieldSet; +import gr.cite.tools.fieldset.FieldSet; +import gr.cite.tools.logging.LoggerService; +import gr.cite.tools.logging.MapLogEntry; +import gr.cite.tools.validation.MyValidate; +import org.apache.commons.io.FilenameUtils; +import org.slf4j.LoggerFactory; +import org.springframework.context.MessageSource; +import org.springframework.context.i18n.LocaleContextHolder; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.InputStreamResource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import javax.management.InvalidApplicationException; +import java.io.IOException; +import java.net.URLConnection; +import java.time.Duration; +import java.util.*; + +import static eu.eudat.authorization.AuthorizationFlags.Public; + +@RestController +@RequestMapping(path = "api/storage-file") +public class StorageFileController { + + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(StorageFileController.class)); + private final AuditService auditService; + private final QueryFactory queryFactory; + private final MessageSource messageSource; + private final StorageFileService storageFileService; + private final StorageFileProperties config; + private final UserScope userScope; + private final AuthorizationService authorizationService; + private final ConventionService conventionService; + public StorageFileController( + AuditService auditService, + QueryFactory queryFactory, + MessageSource messageSource, + StorageFileService storageFileService, + StorageFileProperties config, + UserScope userScope, + AuthorizationService authorizationService, ConventionService conventionService) { + this.auditService = auditService; + this.queryFactory = queryFactory; + this.messageSource = messageSource; + this.storageFileService = storageFileService; + this.config = config; + this.userScope = userScope; + this.authorizationService = authorizationService; + this.conventionService = conventionService; + } + + + @PostMapping("upload-temp-files") + @Transactional + public List uploadTempFiles(@RequestParam("files") MultipartFile[] files) throws IOException { + logger.debug("upload temp files"); + + this.authorizationService.authorizeForce(Permission.EditStorageFile); + + List addedFiles = new ArrayList<>(); + for (MultipartFile file : files) { + StorageFilePersist storageFilePersist = new StorageFilePersist(); + storageFilePersist.setName(FilenameUtils.removeExtension(file.getName())); + storageFilePersist.setExtension(FilenameUtils.getExtension(file.getName())); + storageFilePersist.setMimeType(URLConnection.guessContentTypeFromName(file.getName())); + storageFilePersist.setOwnerId(this.userScope.getUserIdSafe()); + storageFilePersist.setStorageType(StorageType.Temp); + storageFilePersist.setLifetime(Duration.ofSeconds(this.config.getTempStoreLifetimeSeconds())); + StorageFile persisted = this.storageFileService.persistBytes(storageFilePersist, file.getBytes(), new BaseFieldSet(StorageFile._id, StorageFile._name)); + + addedFiles.add(persisted); + } + this.auditService.track(AuditableAction.StorageFile_Upload, "models", addedFiles); + + return addedFiles; + } + + @GetMapping("{id}") + public ResponseEntity get(@PathVariable("id") UUID id) throws MyApplicationException, MyForbiddenException, MyNotFoundException { + logger.debug(new MapLogEntry("download" ).And("id", id)); + + this.authorizationService.authorizeForce(Permission.BrowseStorageFile); + + StorageFileEntity storageFile = this.queryFactory.query(StorageFileQuery.class).ids(id).firstAs(new BaseFieldSet().ensure(StorageFile._createdAt, StorageFile._fullName, StorageFile._mimeType)); + if (storageFile == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, StorageFile.class.getSimpleName()}, LocaleContextHolder.getLocale())); + + byte[] file = this.storageFileService.readAsBytesSafe(id); + if (file == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, StorageFile.class.getSimpleName()}, LocaleContextHolder.getLocale())); + + this.auditService.track(AuditableAction.StorageFile_Download, Map.ofEntries( + new AbstractMap.SimpleEntry("id", id) + )); + + String contentType = storageFile.getMimeType(); + if (this.conventionService.isNullOrEmpty(contentType)) contentType = MediaType.APPLICATION_OCTET_STREAM_VALUE; + + return ResponseEntity.ok() + .contentType(MediaType.valueOf(contentType)) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + storageFile.getName() + (storageFile.getExtension().startsWith(".") ? "" : ".") + storageFile.getExtension() + "\"") + .body(new ByteArrayResource(file)); + } + +} diff --git a/dmp-backend/web/src/main/resources/config/application.yml b/dmp-backend/web/src/main/resources/config/application.yml index 6f847d3b8..0fce2f3d3 100644 --- a/dmp-backend/web/src/main/resources/config/application.yml +++ b/dmp-backend/web/src/main/resources/config/application.yml @@ -19,6 +19,7 @@ spring: optional:classpath:config/swagger.yml[.yml], optional:classpath:config/swagger-${spring.profiles.active}.yml[.yml], optional:file:../config/swagger-${spring.profiles.active}.yml[.yml], optional:classpath:config/deposit.yml[.yml], optional:classpath:config/deposit-${spring.profiles.active}.yml[.yml], optional:file:../config/deposit-${spring.profiles.active}.yml[.yml], optional:classpath:config/errors.yml[.yml], optional:classpath:config/errors-${spring.profiles.active}.yml[.yml], optional:file:../config/errors-${spring.profiles.active}.yml[.yml], + optional:classpath:config/storage.yml[.yml], optional:classpath:config/storage-${spring.profiles.active}.yml[.yml], optional:file:../config/storage-${spring.profiles.active}.yml[.yml], optional:classpath:config/reference-type.yml[.yml], optional:classpath:config/reference-type-${spring.profiles.active}.yml[.yml], optional:file:../config/reference-type-${spring.profiles.active}.yml[.yml], optional:classpath:config/tenant.yml[.yml], optional:classpath:config/tenant-${spring.profiles.active}.yml[.yml], optional:file:../config/tenant-${spring.profiles.active}.yml[.yml] diff --git a/dmp-backend/web/src/main/resources/config/permissions.yml b/dmp-backend/web/src/main/resources/config/permissions.yml index 442863af6..f1ac64c51 100644 --- a/dmp-backend/web/src/main/resources/config/permissions.yml +++ b/dmp-backend/web/src/main/resources/config/permissions.yml @@ -207,6 +207,27 @@ permissions: clients: [ ] allowAnonymous: false allowAuthenticated: false + # StorageFile + BrowseStorageFile: + roles: [ ] + claims: [ ] + clients: [ ] + allowAnonymous: false + allowAuthenticated: true + EditStorageFile: + roles: + - Admin + claims: [ ] + clients: [ ] + allowAnonymous: false + allowAuthenticated: false + DeleteStorageFile: + roles: + - Admin + claims: [ ] + clients: [ ] + allowAnonymous: false + allowAuthenticated: false # DescriptionTemplate BrowseDescriptionTemplate: roles: diff --git a/dmp-backend/web/src/main/resources/config/storage-devel.yml b/dmp-backend/web/src/main/resources/config/storage-devel.yml new file mode 100644 index 000000000..4c9edee77 --- /dev/null +++ b/dmp-backend/web/src/main/resources/config/storage-devel.yml @@ -0,0 +1,7 @@ +storage: + service: + storages: + - type: Temp + basePath: ./storage/temp + - type: Main + basePath: ./storage/main diff --git a/dmp-backend/web/src/main/resources/config/storage.yml b/dmp-backend/web/src/main/resources/config/storage.yml new file mode 100644 index 000000000..649c39240 --- /dev/null +++ b/dmp-backend/web/src/main/resources/config/storage.yml @@ -0,0 +1,6 @@ +storage: + task: + enable: true + intervalSeconds: 600 + service: + tempStoreLifetimeSeconds: 7200 diff --git a/dmp-db-scema/updates/00.01.024_addStorageFile.sql b/dmp-db-scema/updates/00.01.024_addStorageFile.sql new file mode 100644 index 000000000..e4776a4da --- /dev/null +++ b/dmp-db-scema/updates/00.01.024_addStorageFile.sql @@ -0,0 +1,32 @@ +DO $$DECLARE + this_version CONSTANT varchar := '00.01.024'; +BEGIN + PERFORM * FROM "DBVersion" WHERE version = this_version; + IF FOUND THEN RETURN; END IF; + + CREATE TABLE public."StorageFile" + ( + id uuid NOT NULL, + file_ref character varying(100) NOT NULL, + name character varying(250) NOT NULL, + extension character varying(10) NOT NULL, + mime_type character varying(200) NOT NULL, + storage_type smallint NOT NULL, + created_at timestamp without time zone NOT NULL, + purge_at timestamp without time zone, + purged_at timestamp without time zone, + owner uuid, + PRIMARY KEY (id), + FOREIGN KEY (owner) + REFERENCES public."User" (id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE NO ACTION + NOT VALID + ) + WITH ( + OIDS = FALSE + ); + + INSERT INTO public."DBVersion" VALUES ('DMPDB', '00.01.024', '2023-11-20 12:00:00.000000+02', now(), 'Add table StorageFile.'); + +END$$; \ No newline at end of file