Fix File Transformer builder issues and add support for the File Storage Service

dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageType.java

@@ -8,7 +8,8 @@ import java.util.Map;
 public enum StorageType implements DatabaseEnum<Short> {
 
     Temp((short) 0),
-    Main((short) 1);
+    Main((short) 1),
+    Transformer((short)2);
 
     private final Short value;
 

dmp-backend/core/src/main/java/eu/eudat/model/builder/filetransformer/DescriptionTemplateFileTransformerBuilder.java

@@ -3,19 +3,13 @@ package eu.eudat.model.builder.filetransformer;
 import eu.eudat.authorization.AuthorizationFlags;
 import eu.eudat.commons.JsonHandlingService;
 import eu.eudat.commons.XmlHandlingService;
-import eu.eudat.commons.enums.IsActive;
-import eu.eudat.commons.types.description.PropertyDefinitionEntity;
 import eu.eudat.convention.ConventionService;
-import eu.eudat.data.DescriptionEntity;
 import eu.eudat.data.DescriptionTemplateEntity;
-import eu.eudat.file.transformer.models.description.DescriptionFileTransformerModel;
 import eu.eudat.file.transformer.models.descriptiontemplate.DescriptionTemplateFileTransformerModel;
-import eu.eudat.model.DescriptionTemplate;
 import eu.eudat.query.DescriptionTemplateQuery;
 import gr.cite.tools.data.builder.BuilderFactory;
 import gr.cite.tools.data.query.QueryFactory;
 import gr.cite.tools.exception.MyApplicationException;
-import gr.cite.tools.fieldset.BaseFieldSet;
 import gr.cite.tools.logging.LoggerService;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -23,8 +17,10 @@ import org.springframework.beans.factory.config.ConfigurableBeanFactory;
 import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Component;
 
-import java.util.*;
+import java.util.ArrayList;
-import java.util.stream.Collectors;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Optional;
 
 @Component
 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
@@ -68,6 +64,8 @@ public class DescriptionTemplateFileTransformerBuilder extends BaseFileTransform
             m.setId(d.getId());
             m.setLabel(d.getLabel());
             m.setDescription(d.getDescription());
+            m.setLanguage(d.getLanguage());
+            m.setVersion(d.getVersion());
             m.setDefinition(this.builderFactory.builder(DescriptionTemplateDefinitionFileTransformerBuilder.class).authorize(authorize).build(collectDescriptionTemplates(d)));
 
             models.add(new FileTransformerBuilderItemResponse<>(m, d));

dmp-backend/core/src/main/java/eu/eudat/model/builder/filetransformer/DmpFileTransformerBuilder.java

@@ -109,7 +109,7 @@ public class DmpFileTransformerBuilder extends BaseFileTransformerBuilder<DmpFil
             if (descriptionsMap != null && !descriptionsMap.isEmpty() && descriptionsMap.containsKey(d.getId())) m.setDescriptions(descriptionsMap.get(d.getId()));
             if (doiMap != null && !doiMap.isEmpty() && doiMap.containsKey(d.getId())) m.setEntityDois(doiMap.get(d.getId()));
             if (creatorMap != null && !creatorMap.isEmpty() && creatorMap.containsKey(d.getId())) m.setCreator(creatorMap.get(d.getId()));
-            if (blueprintMap != null && !blueprintMap.isEmpty() && blueprintMap.containsKey(d.getId())) m.setBlueprint(blueprintMap.get(d.getId()));
+            if (blueprintMap != null && !blueprintMap.isEmpty() && blueprintMap.containsKey(d.getBlueprintId())) m.setBlueprint(blueprintMap.get(d.getBlueprintId()));
 
             models.add(new FileTransformerBuilderItemResponse<>(m, d));
         }
@@ -171,7 +171,7 @@ public class DmpFileTransformerBuilder extends BaseFileTransformerBuilder<DmpFil
         DmpBlueprintQuery query = this.queryFactory.query(DmpBlueprintQuery.class).isActive(IsActive.Active).authorize(this.authorize).ids(data.stream().map(DmpEntity::getBlueprintId).toList());
         return this.builderFactory.builder(DmpBlueprintFileTransformerBuilder.class).authorize(this.authorize).asMasterKey(query, DmpBlueprintEntity::getId)
                 .entrySet().stream().map(uuidListEntry -> Map.entry(uuidListEntry.getKey(), uuidListEntry.getValue().get(0)))
-                .collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue()));
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
     }
 
     private Map<UUID, List<EntityDoiFileTransformerModel>> collectEntityDois(List<DmpEntity> data) {

dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerService.java

@@ -4,7 +4,7 @@ import eu.eudat.authorization.AuthorizationFlags;
 import eu.eudat.authorization.Permission;
 import eu.eudat.cache.transformer.FileTransformerConfigurationCache;
 import eu.eudat.commons.JsonHandlingService;
-import eu.eudat.commons.enums.DmpUserRole;
+import eu.eudat.commons.enums.StorageType;
 import eu.eudat.configurations.transformer.TransformerProperties;
 import eu.eudat.convention.ConventionService;
 import eu.eudat.file.transformer.interfaces.FileTransformerConfiguration;
@@ -12,33 +12,21 @@ import eu.eudat.file.transformer.models.description.DescriptionFileTransformerMo
 import eu.eudat.file.transformer.models.dmp.DmpFileTransformerModel;
 import eu.eudat.file.transformer.models.misc.FileEnvelope;
 import eu.eudat.file.transformer.models.misc.FileFormat;
-import eu.eudat.model.*;
-import eu.eudat.model.builder.DescriptionBuilder;
-import eu.eudat.model.builder.DmpBuilder;
 import eu.eudat.model.builder.filetransformer.DescriptionFileTransformerBuilder;
 import eu.eudat.model.builder.filetransformer.DmpFileTransformerBuilder;
-import eu.eudat.model.descriptionproperties.PropertyDefinition;
-import eu.eudat.model.descriptiontemplatedefinition.*;
-import eu.eudat.model.descriptiontemplatedefinition.fielddata.AutoCompleteData;
-import eu.eudat.model.descriptiontemplatedefinition.fielddata.AutoCompleteSingleData;
-import eu.eudat.model.descriptiontemplatedefinition.fielddata.BaseFieldData;
-import eu.eudat.model.descriptiontemplatedefinition.fielddata.ComboBoxOption;
-import eu.eudat.model.dmpblueprintdefinition.ExtraField;
-import eu.eudat.model.dmpblueprintdefinition.SystemField;
 import eu.eudat.model.file.TransformerCacheModel;
 import eu.eudat.query.DescriptionQuery;
 import eu.eudat.query.DmpQuery;
 import eu.eudat.query.EntityDoiQuery;
 import eu.eudat.repository.TransformerRepository;
 import eu.eudat.service.entitydoi.EntityDoiService;
+import eu.eudat.service.storage.StorageFileService;
 import gr.cite.commons.web.authz.service.AuthorizationService;
 import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeCacheService;
 import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeFilterFunction;
 import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeModel;
 import gr.cite.tools.data.builder.BuilderFactory;
 import gr.cite.tools.data.query.QueryFactory;
-import gr.cite.tools.fieldset.BaseFieldSet;
-import gr.cite.tools.fieldset.FieldSet;
 import jakarta.persistence.EntityManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -55,7 +43,6 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URI;
-import java.nio.file.Path;
 import java.util.*;
 
 @Service
@@ -75,10 +62,11 @@ public class FileTransformerService {
     private final JsonHandlingService jsonHandlingService;
     private final QueryFactory queryFactory;
     private final BuilderFactory builderFactory;
+    private final StorageFileService storageFileService;
 
 
     @Autowired
-    public FileTransformerService(TransformerProperties transformerProperties, TokenExchangeCacheService tokenExchangeCacheService, FileTransformerConfigurationCache fileTransformerConfigurationCache, WebClient.Builder builder, EntityManager entityManager, AuthorizationService authorizationService, ConventionService conventionService, Environment environment, DmpQuery dmpQuery, EntityDoiQuery doiQuery, EntityDoiService doiService, ApplicationContext applicationContext, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, BuilderFactory builderFactory) {
+    public FileTransformerService(TransformerProperties transformerProperties, TokenExchangeCacheService tokenExchangeCacheService, FileTransformerConfigurationCache fileTransformerConfigurationCache, WebClient.Builder builder, EntityManager entityManager, AuthorizationService authorizationService, ConventionService conventionService, Environment environment, DmpQuery dmpQuery, EntityDoiQuery doiQuery, EntityDoiService doiService, ApplicationContext applicationContext, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, BuilderFactory builderFactory, StorageFileService storageFileService) {
         this.transformerProperties = transformerProperties;
         this.tokenExchangeCacheService = tokenExchangeCacheService;
         this.fileTransformerConfigurationCache = fileTransformerConfigurationCache;
@@ -91,6 +79,7 @@ public class FileTransformerService {
         this.jsonHandlingService = jsonHandlingService;
         this.queryFactory = queryFactory;
         this.builderFactory = builderFactory;
+        this.storageFileService = storageFileService;
         this.clients = new HashMap<>();
     }
 
@@ -306,9 +295,10 @@ public class FileTransformerService {
         DmpFileTransformerModel dmpFileTransformerModel = this.builderFactory.builder(DmpFileTransformerBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(query.first());
         FileEnvelope fileEnvelope = repository.exportDmp(dmpFileTransformerModel, format);
         eu.eudat.model.file.FileEnvelope result = new eu.eudat.model.file.FileEnvelope();
+        byte[] data = storageFileService.readByFileRefAsBytesSafe(fileEnvelope.getFile(), StorageType.Transformer);
         File temp = new File(environment.getProperty("path.path") + UUID.randomUUID());
         try (FileOutputStream fos = new FileOutputStream(temp)) {
-            fos.write(fileEnvelope.getFile());
+            fos.write(data);
         }
         result.setFile(temp);
         result.setFilename(fileEnvelope.getFilename());
@@ -438,9 +428,10 @@ public class FileTransformerService {
         DescriptionFileTransformerModel descriptionFileTransformerModel = this.builderFactory.builder(DescriptionFileTransformerBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(query.first());
         FileEnvelope fileEnvelope = repository.exportDescription(descriptionFileTransformerModel, format);
         eu.eudat.model.file.FileEnvelope result = new eu.eudat.model.file.FileEnvelope();
+        byte[] data = this.storageFileService.readByFileRefAsBytesSafe(fileEnvelope.getFile(), StorageType.Transformer);
         File temp = new File(environment.getProperty("path.path") + UUID.randomUUID());
         try (FileOutputStream fos = new FileOutputStream(temp)) {
-            fos.write(fileEnvelope.getFile());
+            fos.write(data);
         }
         result.setFile(temp);
         result.setFilename(fileEnvelope.getFilename());

dmp-backend/web/src/main/resources/config/storage-devel.yml

@@ -6,6 +6,8 @@ storage:
         basePath: ./storage/temp
       - type: Main
         basePath: ./storage/main
+      - type: Transformer
+        basePath: ${TRANSFORMER_BASE_PATH}
     static-files:
       externalUrls: externalUrls/ExternalUrls.xml
       semantics: Semantics.json