From 403e618d04999d711ef5d70466dd35089a62a801 Mon Sep 17 00:00:00 2001 From: sgiannopoulos Date: Mon, 11 Mar 2024 15:01:56 +0200 Subject: [PATCH] deposit changes --- .../java/eu/eudat/audit/AuditableAction.java | 1 + .../eu/eudat/commons/enums/StorageType.java | 3 +- .../deposit/DepositProperties.java | 18 ++ .../transformer/TransformerProperties.java | 10 +- .../dmp/DmpCommonModelBuilder.java | 7 - .../eudat/model/file/ExportRequestModel.java | 9 + .../main/java/eu/eudat/query/UserQuery.java | 1 + .../service/deposit/DepositServiceImpl.java | 112 ++++++++--- .../java/eu/eudat/service/dmp/DmpService.java | 2 +- .../eu/eudat/service/dmp/DmpServiceImpl.java | 4 +- .../transformer/FileTransformerService.java | 175 +---------------- .../FileTransformerServiceImpl.java | 178 ++++++++++++++++++ .../eu/eudat/controllers/DmpController.java | 14 +- .../FileTransformerController.java | 2 +- .../web/src/main/resources/config/deposit.yml | 2 + .../main/resources/config/storage-devel.yml | 8 +- .../src/main/resources/config/transformer.yml | 3 + 17 files changed, 334 insertions(+), 215 deletions(-) create mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerServiceImpl.java diff --git a/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java b/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java index c8ad61c07..b7e35ea99 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java +++ b/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java @@ -40,6 +40,7 @@ public class AuditableAction { public static final EventId Dmp_Invite_Users = new EventId(5008, "Dmp_Invite_Users"); public static final EventId Dmp_Invite_Accept = new EventId(5009, "Dmp_Invite_Accept"); public static final EventId Dmp_PublicQuery = new EventId(5010, "Dmp_PublicQuery"); + public static final EventId Dmp_Export = new EventId(5011, "Dmp_Export"); public static final EventId Description_Query = new EventId(6000, "Description_Query"); public static final EventId Description_Lookup = new EventId(6001, "Description_Lookup"); diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageType.java b/dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageType.java index 3b72aff20..6525b5abe 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageType.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageType.java @@ -9,7 +9,8 @@ public enum StorageType implements DatabaseEnum { Temp((short) 0), Main((short) 1), - Transformer((short)2); + Transformer((short)2), + Deposit((short)3); private final Short value; diff --git a/dmp-backend/core/src/main/java/eu/eudat/configurations/deposit/DepositProperties.java b/dmp-backend/core/src/main/java/eu/eudat/configurations/deposit/DepositProperties.java index 4ebcded6c..ea695791f 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/configurations/deposit/DepositProperties.java +++ b/dmp-backend/core/src/main/java/eu/eudat/configurations/deposit/DepositProperties.java @@ -26,6 +26,8 @@ public class DepositProperties { private String clientId; private String clientSecret; private String scope; + private String pdfTransformerId; + private String rdaTransformerId; public String getRepositoryId() { return repositoryId; @@ -74,5 +76,21 @@ public class DepositProperties { public void setScope(String scope) { this.scope = scope; } + + public String getPdfTransformerId() { + return pdfTransformerId; + } + + public void setPdfTransformerId(String pdfTransformerId) { + this.pdfTransformerId = pdfTransformerId; + } + + public String getRdaTransformerId() { + return rdaTransformerId; + } + + public void setRdaTransformerId(String rdaTransformerId) { + this.rdaTransformerId = rdaTransformerId; + } } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/configurations/transformer/TransformerProperties.java b/dmp-backend/core/src/main/java/eu/eudat/configurations/transformer/TransformerProperties.java index 7457f28a3..d2d0e9e33 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/configurations/transformer/TransformerProperties.java +++ b/dmp-backend/core/src/main/java/eu/eudat/configurations/transformer/TransformerProperties.java @@ -22,6 +22,7 @@ public class TransformerProperties { public static class TransformerSource { private final String url; + private final String transformerId; private final List codes; private final String issuerUrl; private final String clientId; @@ -29,9 +30,10 @@ public class TransformerProperties { private final String scope; @ConstructorBinding - public TransformerSource(String url, List codes, String issuerUrl, String clientId, String clientSecret, String scope) { + public TransformerSource(String url, String transformerId, List codes, String issuerUrl, String clientId, String clientSecret, String scope) { this.url = url; - this.codes = codes; + this.transformerId = transformerId; + this.codes = codes; this.issuerUrl = issuerUrl; this.clientId = clientId; this.clientSecret = clientSecret; @@ -61,5 +63,9 @@ public class TransformerProperties { public List getCodes() { return codes; } + + public String getTransformerId() { + return transformerId; + } } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/builder/commonmodels/dmp/DmpCommonModelBuilder.java b/dmp-backend/core/src/main/java/eu/eudat/model/builder/commonmodels/dmp/DmpCommonModelBuilder.java index e1e4a0f3c..dc64e93ca 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/model/builder/commonmodels/dmp/DmpCommonModelBuilder.java +++ b/dmp-backend/core/src/main/java/eu/eudat/model/builder/commonmodels/dmp/DmpCommonModelBuilder.java @@ -49,7 +49,6 @@ public class DmpCommonModelBuilder extends BaseCommonModelBuilder authorize = EnumSet.of(AuthorizationFlags.None); @@ -79,11 +78,6 @@ public class DmpCommonModelBuilder extends BaseCommonModelBuilder (short)1) m.setPreviousDOI(this.getPreviousDOI(d.getGroupId(), d.getId())); switch (d.getAccessType()){ case Public -> m.setAccessType(DmpAccessType.Public); diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/file/ExportRequestModel.java b/dmp-backend/core/src/main/java/eu/eudat/model/file/ExportRequestModel.java index 6e782b6db..ab6dceca7 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/model/file/ExportRequestModel.java +++ b/dmp-backend/core/src/main/java/eu/eudat/model/file/ExportRequestModel.java @@ -5,6 +5,7 @@ import java.util.UUID; public class ExportRequestModel { private UUID id; + private String repositoryId; private String format; public UUID getId() { @@ -22,4 +23,12 @@ public class ExportRequestModel { public void setFormat(String format) { this.format = format; } + + public String getRepositoryId() { + return repositoryId; + } + + public void setRepositoryId(String repositoryId) { + this.repositoryId = repositoryId; + } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/query/UserQuery.java b/dmp-backend/core/src/main/java/eu/eudat/query/UserQuery.java index dee5c6a60..099a3ee5f 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/query/UserQuery.java +++ b/dmp-backend/core/src/main/java/eu/eudat/query/UserQuery.java @@ -150,6 +150,7 @@ public class UserQuery extends QueryBase { if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseUser)) return null; UUID userId; if (this.authorize.contains(AuthorizationFlags.Owner)) userId = this.userScope.getUserIdSafe(); + if (this.authorize.contains(AuthorizationFlags.Public)) userId = this.userScope.getUserIdSafe(); else userId = null; List predicates = new ArrayList<>(); diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/deposit/DepositServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/deposit/DepositServiceImpl.java index 47e0382a7..d5e642620 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/deposit/DepositServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/deposit/DepositServiceImpl.java @@ -3,19 +3,34 @@ package eu.eudat.service.deposit; import eu.eudat.authorization.AuthorizationFlags; import eu.eudat.authorization.Permission; import eu.eudat.commonmodels.models.FileEnvelopeModel; +import eu.eudat.commonmodels.models.description.DescriptionModel; +import eu.eudat.commonmodels.models.description.FieldModel; +import eu.eudat.commonmodels.models.description.PropertyDefinitionFieldSetItemModel; +import eu.eudat.commonmodels.models.description.PropertyDefinitionFieldSetModel; import eu.eudat.commonmodels.models.dmp.DmpModel; +import eu.eudat.commons.enums.StorageType; +import eu.eudat.commons.scope.user.UserScope; import eu.eudat.configurations.deposit.DepositProperties; +import eu.eudat.convention.ConventionService; +import eu.eudat.data.DescriptionTemplateEntity; import eu.eudat.data.DmpEntity; import eu.eudat.depositinterface.repository.DepositClient; import eu.eudat.depositinterface.repository.DepositConfiguration; +import eu.eudat.model.DescriptionTemplate; import eu.eudat.model.EntityDoi; +import eu.eudat.model.StorageFile; import eu.eudat.model.builder.commonmodels.DepositConfigurationBuilder; import eu.eudat.model.builder.commonmodels.dmp.DmpCommonModelBuilder; +import eu.eudat.model.persist.StorageFilePersist; import eu.eudat.model.persist.deposit.DepositAuthenticateRequest; import eu.eudat.model.persist.deposit.DepositRequest; import eu.eudat.model.persist.EntityDoiPersist; +import eu.eudat.query.DescriptionTemplateQuery; import eu.eudat.query.DmpQuery; import eu.eudat.service.entitydoi.EntityDoiService; +import eu.eudat.service.storage.StorageFileProperties; +import eu.eudat.service.storage.StorageFileService; +import eu.eudat.service.transformer.FileTransformerService; import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeCacheService; import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeFilterFunction; @@ -23,7 +38,10 @@ import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeModel; import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.exception.MyNotFoundException; +import gr.cite.tools.fieldset.BaseFieldSet; import gr.cite.tools.fieldset.FieldSet; +import gr.cite.tools.validation.ValidatorFactory; +import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -32,11 +50,12 @@ import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.stereotype.Service; import org.springframework.web.reactive.function.client.WebClient; +import java.io.IOException; import java.net.URI; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.net.URLConnection; +import java.time.Duration; +import java.util.*; +import java.util.stream.Collectors; @Service public class DepositServiceImpl implements DepositService { @@ -52,7 +71,12 @@ public class DepositServiceImpl implements DepositService { private final MessageSource messageSource; private final BuilderFactory builderFactory; private final DepositConfigurationCacheService depositConfigurationCacheService; - + private final FileTransformerService fileTransformerService; + private final StorageFileService storageFileService; + private final UserScope userScope; + private final ValidatorFactory validatorFactory; + private final StorageFileProperties storageFileProperties; + private final ConventionService conventionService; @Autowired public DepositServiceImpl(DepositProperties depositProperties, TokenExchangeCacheService tokenExchangeCacheService, @@ -61,7 +85,7 @@ public class DepositServiceImpl implements DepositService { EntityDoiService doiService, QueryFactory queryFactory, MessageSource messageSource, - BuilderFactory builderFactory, DepositConfigurationCacheService depositConfigurationCacheService) { + BuilderFactory builderFactory, DepositConfigurationCacheService depositConfigurationCacheService, FileTransformerService fileTransformerService, StorageFileService storageFileService, UserScope userScope, ValidatorFactory validatorFactory, StorageFileProperties storageFileProperties, ConventionService conventionService) { this.depositProperties = depositProperties; this.tokenExchangeCacheService = tokenExchangeCacheService; this.authorizationService = authorizationService; @@ -71,6 +95,12 @@ public class DepositServiceImpl implements DepositService { this.messageSource = messageSource; this.builderFactory = builderFactory; this.depositConfigurationCacheService = depositConfigurationCacheService; + this.fileTransformerService = fileTransformerService; + this.storageFileService = storageFileService; + this.userScope = userScope; + this.validatorFactory = validatorFactory; + this.storageFileProperties = storageFileProperties; + this.conventionService = conventionService; this.clients = new HashMap<>(); } @@ -130,29 +160,48 @@ public class DepositServiceImpl implements DepositService { //GK: Forth make the required files to be uploaded with the deposit //TODO: Properly create required files - FileEnvelopeModel docEnvelope = new FileEnvelopeModel(); + DepositProperties.DepositSource source = depositProperties.getSources().stream().filter(depositSource -> depositSource.getRepositoryId().equals(dmpDepositModel.getRepositoryId())).findFirst().orElse(null); + if (source == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{dmpDepositModel.getRepositoryId(), DepositProperties.DepositSource.class.getSimpleName()}, LocaleContextHolder.getLocale())); + + eu.eudat.model.file.FileEnvelope pdfFile = this.fileTransformerService.exportDmp(dmpEntity.getId(), source.getPdfTransformerId(),"pdf"); + eu.eudat.model.file.FileEnvelope rda = this.fileTransformerService.exportDmp(dmpEntity.getId(), source.getRdaTransformerId(),"json"); + FileEnvelopeModel pdfEnvelope = new FileEnvelopeModel(); FileEnvelopeModel jsonEnvelope = new FileEnvelopeModel(); - FileEnvelopeModel zip = new FileEnvelopeModel(); - //TODO: Create Files -// File zip = new File(environment.getProperty("temp.temp") + UUID.randomUUID() + ".zip"); -// try { -// docEnvelope.setFilename("test.docx"); -// docEnvelope.setFile(this.storageFileService.getH2020TemplateFile()); -// File pdfFile = PDFUtils.convertToPDF(docEnvelope, environment); -// pdfEnvelope.setFilename("test.pdf"); -// pdfEnvelope.setFile(pdfFile); -// File jsonFile = new File(this.environment.getProperty("temp.temp") + UUID.randomUUID() + ".json"); -// jsonEnvelope.setFilename("test.json"); -// jsonEnvelope.setFile(jsonFile); -// -// } catch (IOException e) { -// logger.error(e.getMessage(), e); -// } - + + pdfEnvelope.setFilename(pdfFile.getFilename()); + jsonEnvelope.setFilename(rda.getFilename()); + if (!depositClient.getConfiguration().isUseSharedStorage()){ + pdfEnvelope.setFile(pdfFile.getFile()); + jsonEnvelope.setFile(rda.getFile()); + } else { + pdfEnvelope.setFileRef(this.addFileToSharedStorage(pdfFile)); + jsonEnvelope.setFileRef(this.addFileToSharedStorage(rda)); + } + //GK: Fifth Transform them to the DepositModel DmpModel depositModel = this.builderFactory.builder(DmpCommonModelBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic) - .setRepositoryId(dmpDepositModel.getRepositoryId()).setPdfFile(pdfEnvelope).setRdaJsonFile(jsonEnvelope).setSupportingFilesZip(zip).build(dmpEntity); + .setRepositoryId(dmpDepositModel.getRepositoryId()).setPdfFile(pdfEnvelope).setRdaJsonFile(jsonEnvelope).build(dmpEntity); + + //TODO +// if (depositModel != null && !this.conventionService.isListNullOrEmpty(depositModel.getDescriptions())){ +// for (DescriptionModel descriptionModel :depositModel.getDescriptions()){ +// DescriptionTemplateEntity descriptionTemplate = this.queryFactory.query(DescriptionTemplateQuery.class).ids(descriptionModel.getDescriptionTemplate().getId()).firstAs(new BaseFieldSet().ensure(DescriptionTemplate._definition)); +// if (descriptionTemplate == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{descriptionModel.getDescriptionTemplate().getId(), DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale())); +// +// if (descriptionModel.getProperties() != null && descriptionModel.getProperties().getFieldSets() != null){ +// for(PropertyDefinitionFieldSetItemModel propertyDefinitionFieldSetItemModel : descriptionModel.getProperties().getFieldSets().values().stream().filter(x-> x.getItems() != null).map(PropertyDefinitionFieldSetModel::getItems).flatMap(Collection::stream).toList()){ +// if (propertyDefinitionFieldSetItemModel.getFields() != null){ +// for(Map.Entry entrySet : propertyDefinitionFieldSetItemModel.getFields().entrySet()){ +// +// } +// } +// } +// } +// +// } +// +// } //GK: Sixth Perform the deposit String doi = depositClient.deposit(depositModel, dmpDepositModel.getAccessToken()); @@ -165,6 +214,19 @@ public class DepositServiceImpl implements DepositService { doiPersist.setEntityId(dmpEntity.getId()); return doiService.persist(doiPersist, dmpDepositModel.getProject()); } + + private String addFileToSharedStorage(eu.eudat.model.file.FileEnvelope file) throws IOException { + StorageFilePersist storageFilePersist = new StorageFilePersist(); + storageFilePersist.setName(FilenameUtils.removeExtension(file.getFilename())); + storageFilePersist.setExtension(FilenameUtils.getExtension(file.getFilename())); + storageFilePersist.setMimeType(URLConnection.guessContentTypeFromName(file.getFilename())); + storageFilePersist.setOwnerId(this.userScope.getUserIdSafe()); + storageFilePersist.setStorageType(StorageType.Temp); + storageFilePersist.setLifetime(Duration.ofSeconds(this.storageFileProperties.getTempStoreLifetimeSeconds())); //TODO + this.validatorFactory.validator(StorageFilePersist.StorageFilePersistValidator.class).validateForce(storageFilePersist); + StorageFile persisted = this.storageFileService.persistBytes(storageFilePersist, file.getFile(), new BaseFieldSet(StorageFile._id, StorageFile._fileRef)); + return persisted.getFileRef(); + } @Override public String getLogo(String repositoryId) { diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpService.java b/dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpService.java index 1e9f8df1f..76e92b64b 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpService.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpService.java @@ -34,7 +34,7 @@ public interface DmpService { List assignUsers(UUID dmp, List model, FieldSet fields) throws InvalidApplicationException, IOException; Dmp removeUser(DmpUserRemovePersist model, FieldSet fields) throws InvalidApplicationException, IOException; - ResponseEntity export(UUID id, String exportType) throws InvalidApplicationException, IOException; + ResponseEntity export(UUID id, String transformerId, String exportType) throws InvalidApplicationException, IOException; void inviteUsers(UUID id, List users) throws InvalidApplicationException, JAXBException, IOException; diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpServiceImpl.java index 63a9f01bc..687dbb00c 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpServiceImpl.java @@ -518,10 +518,10 @@ public class DmpServiceImpl implements DmpService { } @Override - public ResponseEntity export(UUID id, String exportType) throws InvalidApplicationException, IOException { + public ResponseEntity export(UUID id, String transformerId, String exportType) throws InvalidApplicationException, IOException { HttpHeaders headers = new HttpHeaders(); - FileEnvelope fileEnvelope = this.fileTransformerService.exportDmp(id, exportType); + FileEnvelope fileEnvelope = this.fileTransformerService.exportDmp(id, transformerId, exportType); headers.add("Content-Disposition", "attachment;filename=" + fileEnvelope.getFilename()); byte[] data = fileEnvelope.getFile(); headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerService.java b/dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerService.java index 265d53c9c..674259823 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerService.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerService.java @@ -1,177 +1,14 @@ package eu.eudat.service.transformer; -import eu.eudat.authorization.AuthorizationFlags; -import eu.eudat.authorization.Permission; -import eu.eudat.cache.transformer.FileTransformerConfigurationCache; -import eu.eudat.commonmodels.models.FileEnvelopeModel; -import eu.eudat.commonmodels.models.description.DescriptionModel; -import eu.eudat.commonmodels.models.dmp.DmpModel; -import eu.eudat.commons.enums.StorageType; -import eu.eudat.configurations.transformer.TransformerProperties; -import eu.eudat.file.transformer.interfaces.FileTransformerConfiguration; import eu.eudat.file.transformer.models.misc.FileFormat; -import eu.eudat.model.Description; -import eu.eudat.model.Dmp; -import eu.eudat.model.builder.commonmodels.description.DescriptionCommonModelBuilder; -import eu.eudat.model.builder.commonmodels.dmp.DmpCommonModelBuilder; -import eu.eudat.model.file.TransformerCacheModel; -import eu.eudat.query.DescriptionQuery; -import eu.eudat.query.DmpQuery; -import eu.eudat.repository.TransformerRepository; -import eu.eudat.service.storage.StorageFileService; -import gr.cite.commons.web.authz.service.AuthorizationService; -import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeCacheService; -import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeFilterFunction; -import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeModel; -import gr.cite.tools.data.builder.BuilderFactory; -import gr.cite.tools.data.query.QueryFactory; -import gr.cite.tools.exception.MyNotFoundException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.MessageSource; -import org.springframework.context.i18n.LocaleContextHolder; -import org.springframework.stereotype.Service; -import org.springframework.web.reactive.function.client.ExchangeFilterFunction; -import org.springframework.web.reactive.function.client.WebClient; -import reactor.core.publisher.Mono; -import javax.management.InvalidApplicationException; -import java.io.IOException; -import java.net.URI; -import java.util.*; +import java.util.List; +import java.util.UUID; -@Service -public class FileTransformerService { - private static final Logger logger = LoggerFactory.getLogger(FileTransformerService.class); +public interface FileTransformerService { + List getAvailableExportFileFormats(); - private final TransformerProperties transformerProperties; - private final Map clients; - private final TokenExchangeCacheService tokenExchangeCacheService; - private final FileTransformerConfigurationCache fileTransformerConfigurationCache; - private final AuthorizationService authorizationService; - private final WebClient.Builder webClientBuilder; - private final QueryFactory queryFactory; - private final BuilderFactory builderFactory; - private final StorageFileService storageFileService; - private final MessageSource messageSource; + eu.eudat.model.file.FileEnvelope exportDmp(UUID dmpId, String repositoryId, String format); - @Autowired - public FileTransformerService(TransformerProperties transformerProperties, TokenExchangeCacheService tokenExchangeCacheService, FileTransformerConfigurationCache fileTransformerConfigurationCache, WebClient.Builder builder, AuthorizationService authorizationService, - QueryFactory queryFactory, BuilderFactory builderFactory, StorageFileService storageFileService, MessageSource messageSource) { - this.transformerProperties = transformerProperties; - this.tokenExchangeCacheService = tokenExchangeCacheService; - this.fileTransformerConfigurationCache = fileTransformerConfigurationCache; - this.authorizationService = authorizationService; - this.webClientBuilder = builder; - this.queryFactory = queryFactory; - this.builderFactory = builderFactory; - this.storageFileService = storageFileService; - this.messageSource = messageSource; - this.clients = new HashMap<>(); - } - - private TransformerRepository getRepository(String repoId) { - if (this.clients.containsKey(repoId)) return this.clients.get(repoId); - - //GK: It's register time - TransformerProperties.TransformerSource source = transformerProperties.getSources().stream().filter(depositSource -> depositSource.getCodes().contains(repoId)).findFirst().orElse(null); - if (source != null) { - String host = URI.create(source.getUrl()).getHost(); - TokenExchangeModel tokenExchangeModel = new TokenExchangeModel(host + "_" + source.getClientId(), source.getIssuerUrl(), source.getClientId(), source.getClientSecret(), source.getScope()); - TokenExchangeFilterFunction tokenExchangeFilterFunction = new TokenExchangeFilterFunction(this.tokenExchangeCacheService, tokenExchangeModel); - TransformerRepository repository = new TransformerRepository(webClientBuilder.baseUrl(source.getUrl() + "/api/file-transformer").filters(exchangeFilterFunctions -> { - exchangeFilterFunctions.add(tokenExchangeFilterFunction); - exchangeFilterFunctions.add(logRequest()); - }).build()); - source.getCodes().forEach(code -> this.clients.put(code, repository)); - return repository; - } - return null; - } - - - public List getAvailableExportFileFormats() { - List formats = new ArrayList<>(); - List configurations = this.getAvailableConfigurations(); - if(configurations != null){ - for (FileTransformerConfiguration configuration : configurations){ - if (configuration != null && configuration.getExportVariants() != null) formats.addAll(configuration.getExportVariants()); - } - } - return formats; - } - - private List getAvailableConfigurations() { - TransformerCacheModel configs = fileTransformerConfigurationCache.lookup("base"); - if (configs == null) { - List configurations = new ArrayList<>(); - //GK: So much for lazy loading - List repositories = transformerProperties.getSources().stream().map(depositSource -> getRepository(depositSource.getCodes().getFirst())).toList(); - - repositories = new ArrayList<>(repositories); - repositories.forEach((client) -> { - try { - FileTransformerConfiguration repositoryConfig = client.getConfiguration(); - if (repositoryConfig != null) { - configurations.add(repositoryConfig); - } - } catch (Exception e) { - logger.warn(e.getLocalizedMessage(), e); - } - }); - - configs = new TransformerCacheModel(configurations); - this.fileTransformerConfigurationCache.put("base", configs); - } - - return configs.getConfigurations(); - } - - public eu.eudat.model.file.FileEnvelope exportDmp(UUID dmpId, String format) throws InvalidApplicationException, IOException { - this.authorizationService.authorize(Permission.EditDmp); - //GK: First get the right client - TransformerRepository repository = getRepository(format); - if (repository == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{format, TransformerRepository.class.getSimpleName()}, LocaleContextHolder.getLocale())); - //GK: Second get the Target Data Management Plan - DmpQuery query = this.queryFactory.query(DmpQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(dmpId); - DmpModel dmpFileTransformerModel = this.builderFactory.builder(DmpCommonModelBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(query.first()); - if (dmpFileTransformerModel == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{dmpId, Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale())); - - FileEnvelopeModel fileEnvelope = repository.exportDmp(dmpFileTransformerModel, format); - eu.eudat.model.file.FileEnvelope result = new eu.eudat.model.file.FileEnvelope(); - - byte[] data = repository.getConfiguration().isUseSharedStorage() ? storageFileService.readByFileRefAsBytesSafe(fileEnvelope.getFileRef(), StorageType.Transformer) : fileEnvelope.getFile(); - result.setFile(data); - result.setFilename(fileEnvelope.getFilename()); - return result; - } - - public eu.eudat.model.file.FileEnvelope exportDescription(UUID descriptionId, String format) { - this.authorizationService.authorize(Permission.EditDmp); - //GK: First get the right client - TransformerRepository repository = getRepository(format); - if (repository == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{format, TransformerRepository.class.getSimpleName()}, LocaleContextHolder.getLocale())); - - //GK: Second get the Target Data Management Plan - DescriptionQuery query = this.queryFactory.query(DescriptionQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(descriptionId); - DescriptionModel descriptionFileTransformerModel = this.builderFactory.builder(DescriptionCommonModelBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(query.first()); - if (descriptionFileTransformerModel == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{descriptionId, Description.class.getSimpleName()}, LocaleContextHolder.getLocale())); - - FileEnvelopeModel fileEnvelope = repository.exportDescription(descriptionFileTransformerModel, format); - eu.eudat.model.file.FileEnvelope result = new eu.eudat.model.file.FileEnvelope(); - byte[] data = repository.getConfiguration().isUseSharedStorage() ? storageFileService.readByFileRefAsBytesSafe(fileEnvelope.getFileRef(), StorageType.Transformer) : fileEnvelope.getFile(); //TODO: shared storage should be per repository - result.setFile(data); - result.setFilename(fileEnvelope.getFilename()); - return result; - } - - // This method returns filter function which will log request data - private static ExchangeFilterFunction logRequest() { - return ExchangeFilterFunction.ofRequestProcessor(clientRequest -> { - logger.info("Request: {} {}", clientRequest.method(), clientRequest.url()); - clientRequest.headers().forEach((name, values) -> values.forEach(value -> logger.info("{}={}", name, value))); - return Mono.just(clientRequest); - }); - } + eu.eudat.model.file.FileEnvelope exportDescription(UUID descriptionId, String format); } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerServiceImpl.java new file mode 100644 index 000000000..c3a469e2c --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerServiceImpl.java @@ -0,0 +1,178 @@ +package eu.eudat.service.transformer; + +import eu.eudat.authorization.AuthorizationFlags; +import eu.eudat.authorization.Permission; +import eu.eudat.cache.transformer.FileTransformerConfigurationCache; +import eu.eudat.commonmodels.models.FileEnvelopeModel; +import eu.eudat.commonmodels.models.description.DescriptionModel; +import eu.eudat.commonmodels.models.dmp.DmpModel; +import eu.eudat.commons.enums.StorageType; +import eu.eudat.configurations.transformer.TransformerProperties; +import eu.eudat.file.transformer.interfaces.FileTransformerConfiguration; +import eu.eudat.file.transformer.models.misc.FileFormat; +import eu.eudat.model.Description; +import eu.eudat.model.Dmp; +import eu.eudat.model.builder.commonmodels.description.DescriptionCommonModelBuilder; +import eu.eudat.model.builder.commonmodels.dmp.DmpCommonModelBuilder; +import eu.eudat.model.file.TransformerCacheModel; +import eu.eudat.query.DescriptionQuery; +import eu.eudat.query.DmpQuery; +import eu.eudat.repository.TransformerRepository; +import eu.eudat.service.storage.StorageFileService; +import gr.cite.commons.web.authz.service.AuthorizationService; +import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeCacheService; +import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeFilterFunction; +import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeModel; +import gr.cite.tools.data.builder.BuilderFactory; +import gr.cite.tools.data.query.QueryFactory; +import gr.cite.tools.exception.MyNotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.MessageSource; +import org.springframework.context.i18n.LocaleContextHolder; +import org.springframework.stereotype.Service; +import org.springframework.web.reactive.function.client.ExchangeFilterFunction; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.core.publisher.Mono; + +import java.net.URI; +import java.util.*; + +@Service +public class FileTransformerServiceImpl implements FileTransformerService { + private static final Logger logger = LoggerFactory.getLogger(FileTransformerServiceImpl.class); + + private final TransformerProperties transformerProperties; + private final Map clients; + private final TokenExchangeCacheService tokenExchangeCacheService; + private final FileTransformerConfigurationCache fileTransformerConfigurationCache; + private final AuthorizationService authorizationService; + private final WebClient.Builder webClientBuilder; + private final QueryFactory queryFactory; + private final BuilderFactory builderFactory; + private final StorageFileService storageFileService; + private final MessageSource messageSource; + + @Autowired + public FileTransformerServiceImpl(TransformerProperties transformerProperties, TokenExchangeCacheService tokenExchangeCacheService, FileTransformerConfigurationCache fileTransformerConfigurationCache, WebClient.Builder builder, AuthorizationService authorizationService, + QueryFactory queryFactory, BuilderFactory builderFactory, StorageFileService storageFileService, MessageSource messageSource) { + this.transformerProperties = transformerProperties; + this.tokenExchangeCacheService = tokenExchangeCacheService; + this.fileTransformerConfigurationCache = fileTransformerConfigurationCache; + this.authorizationService = authorizationService; + this.webClientBuilder = builder; + this.queryFactory = queryFactory; + this.builderFactory = builderFactory; + this.storageFileService = storageFileService; + this.messageSource = messageSource; + this.clients = new HashMap<>(); + } + + private TransformerRepository getRepository(String repoId) { + if (this.clients.containsKey(repoId)) return this.clients.get(repoId); + + //GK: It's register time + TransformerProperties.TransformerSource source = transformerProperties.getSources().stream().filter(depositSource -> depositSource.getCodes().contains(repoId)).findFirst().orElse(null); + if (source != null) { + String host = URI.create(source.getUrl()).getHost(); + TokenExchangeModel tokenExchangeModel = new TokenExchangeModel(host + "_" + source.getClientId(), source.getIssuerUrl(), source.getClientId(), source.getClientSecret(), source.getScope()); + TokenExchangeFilterFunction tokenExchangeFilterFunction = new TokenExchangeFilterFunction(this.tokenExchangeCacheService, tokenExchangeModel); + TransformerRepository repository = new TransformerRepository(webClientBuilder.baseUrl(source.getUrl() + "/api/file-transformer").filters(exchangeFilterFunctions -> { + exchangeFilterFunctions.add(tokenExchangeFilterFunction); + exchangeFilterFunctions.add(logRequest()); + }).build()); + source.getCodes().forEach(code -> this.clients.put(code, repository)); + return repository; + } + return null; + } + + + @Override + public List getAvailableExportFileFormats() { + List formats = new ArrayList<>(); + List configurations = this.getAvailableConfigurations(); + if(configurations != null){ + for (FileTransformerConfiguration configuration : configurations){ + if (configuration != null && configuration.getExportVariants() != null) formats.addAll(configuration.getExportVariants()); + } + } + return formats; + } + + private List getAvailableConfigurations() { + TransformerCacheModel configs = fileTransformerConfigurationCache.lookup("base"); + if (configs == null) { + List configurations = new ArrayList<>(); + //GK: So much for lazy loading + List repositories = transformerProperties.getSources().stream().map(depositSource -> getRepository(depositSource.getCodes().getFirst())).toList(); + + repositories = new ArrayList<>(repositories); + repositories.forEach((client) -> { + try { + FileTransformerConfiguration repositoryConfig = client.getConfiguration(); + if (repositoryConfig != null) { + configurations.add(repositoryConfig); + } + } catch (Exception e) { + logger.warn(e.getLocalizedMessage(), e); + } + }); + + configs = new TransformerCacheModel(configurations); + this.fileTransformerConfigurationCache.put("base", configs); + } + + return configs.getConfigurations(); + } + + @Override + public eu.eudat.model.file.FileEnvelope exportDmp(UUID dmpId, String repositoryId, String format) { + this.authorizationService.authorize(Permission.EditDmp); + //GK: First get the right client + TransformerRepository repository = getRepository(repositoryId); + if (repository == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{repositoryId, TransformerRepository.class.getSimpleName()}, LocaleContextHolder.getLocale())); + //GK: Second get the Target Data Management Plan + DmpQuery query = this.queryFactory.query(DmpQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(dmpId); + DmpModel dmpFileTransformerModel = this.builderFactory.builder(DmpCommonModelBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(query.first()); + if (dmpFileTransformerModel == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{dmpId, Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale())); + + FileEnvelopeModel fileEnvelope = repository.exportDmp(dmpFileTransformerModel, format); + eu.eudat.model.file.FileEnvelope result = new eu.eudat.model.file.FileEnvelope(); + + byte[] data = repository.getConfiguration().isUseSharedStorage() ? storageFileService.readByFileRefAsBytesSafe(fileEnvelope.getFileRef(), StorageType.Transformer) : fileEnvelope.getFile(); + result.setFile(data); + result.setFilename(fileEnvelope.getFilename()); + return result; + } + + @Override + public eu.eudat.model.file.FileEnvelope exportDescription(UUID descriptionId, String format) { + this.authorizationService.authorize(Permission.EditDmp); + //GK: First get the right client + TransformerRepository repository = getRepository(format); + if (repository == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{format, TransformerRepository.class.getSimpleName()}, LocaleContextHolder.getLocale())); + + //GK: Second get the Target Data Management Plan + DescriptionQuery query = this.queryFactory.query(DescriptionQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(descriptionId); + DescriptionModel descriptionFileTransformerModel = this.builderFactory.builder(DescriptionCommonModelBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(query.first()); + if (descriptionFileTransformerModel == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{descriptionId, Description.class.getSimpleName()}, LocaleContextHolder.getLocale())); + + FileEnvelopeModel fileEnvelope = repository.exportDescription(descriptionFileTransformerModel, format); + eu.eudat.model.file.FileEnvelope result = new eu.eudat.model.file.FileEnvelope(); + byte[] data = repository.getConfiguration().isUseSharedStorage() ? storageFileService.readByFileRefAsBytesSafe(fileEnvelope.getFileRef(), StorageType.Transformer) : fileEnvelope.getFile(); //TODO: shared storage should be per repository + result.setFile(data); + result.setFilename(fileEnvelope.getFilename()); + return result; + } + + // This method returns filter function which will log request data + private static ExchangeFilterFunction logRequest() { + return ExchangeFilterFunction.ofRequestProcessor(clientRequest -> { + logger.info("Request: {} {}", clientRequest.method(), clientRequest.url()); + clientRequest.headers().forEach((name, values) -> values.forEach(value -> logger.info("{}={}", name, value))); + return Mono.just(clientRequest); + }); + } +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/DmpController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/DmpController.java index c3e2ad149..5781a0f68 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/DmpController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/DmpController.java @@ -215,11 +215,17 @@ public class DmpController { return new QueryResult<>(persisted); } - @GetMapping("{id}/export/{type}") - public ResponseEntity export(@PathVariable("id") UUID id, @PathVariable("type") String exportType) throws InvalidApplicationException, IOException { - logger.debug(new MapLogEntry("exporting dmp")); + @GetMapping("{id}/export/{transformerId}/{type}") + public ResponseEntity export(@PathVariable("id") UUID id, @PathVariable("transformerId") String transformerId, @PathVariable("type") String exportType) throws InvalidApplicationException, IOException { + logger.debug(new MapLogEntry("exporting dmp").And("id", id).And("transformerId", transformerId).And("exportType", exportType)); - return this.dmpService.export(id, exportType); + ResponseEntity bytes = this.dmpService.export(id, transformerId, exportType); + this.auditService.track(AuditableAction.Dmp_Export, Map.ofEntries( + new AbstractMap.SimpleEntry("id", id), + new AbstractMap.SimpleEntry("transformerId", transformerId), + new AbstractMap.SimpleEntry("exportType", exportType) + )); + return bytes; } @PostMapping("{id}/invite-users") diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/FileTransformerController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/FileTransformerController.java index a520eb020..a06b1642b 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/FileTransformerController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/FileTransformerController.java @@ -52,7 +52,7 @@ public class FileTransformerController { logger.debug(new MapLogEntry("exporting dmp")); HttpHeaders headers = new HttpHeaders(); - FileEnvelope fileEnvelope = this.fileTransformerService.exportDmp(requestModel.getId(), requestModel.getFormat()); + FileEnvelope fileEnvelope = this.fileTransformerService.exportDmp(requestModel.getId(), requestModel.getRepositoryId(), requestModel.getFormat()); headers.add("Content-Disposition", "attachment;filename=" + fileEnvelope.getFilename()); byte[] data = fileEnvelope.getFile(); headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); diff --git a/dmp-backend/web/src/main/resources/config/deposit.yml b/dmp-backend/web/src/main/resources/config/deposit.yml index ea22fd026..84b70f957 100644 --- a/dmp-backend/web/src/main/resources/config/deposit.yml +++ b/dmp-backend/web/src/main/resources/config/deposit.yml @@ -2,6 +2,8 @@ deposit: sources: - url: http://localhost:8082 repositoryId: zenodo + pdfTransformerId: docx-file-transformer + rdaTransformerId: rda-file-transformer issuer-url: ${ZENODO_ISSUER_URI:IDP_APIKEY_ISSUER_URI}/protocol/openid-connect/token client-id: ${ZENODO_DEPOSIT_CLIENT_ID:} client-secret: ${ZENODO_DEPOSIT_CLIENT_SECRET:} diff --git a/dmp-backend/web/src/main/resources/config/storage-devel.yml b/dmp-backend/web/src/main/resources/config/storage-devel.yml index 5bd3e9916..3731cc12c 100644 --- a/dmp-backend/web/src/main/resources/config/storage-devel.yml +++ b/dmp-backend/web/src/main/resources/config/storage-devel.yml @@ -3,11 +3,13 @@ storage: defaultLanguage: en storages: - type: Temp - basePath: ./storage/temp + basePath: ${FILE_STORAGE}/temp - type: Main - basePath: ./storage/main + basePath: ${FILE_STORAGE}/main - type: Transformer - basePath: ${TRANSFORMER_BASE_PATH} + basePath: ${FILE_STORAGE}/transformer + - type: Deposit + basePath: ${FILE_STORAGE}/deposit static-files: externalUrls: dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml semantics: dmp-backend/web/src/main/resources/Semantics.json diff --git a/dmp-backend/web/src/main/resources/config/transformer.yml b/dmp-backend/web/src/main/resources/config/transformer.yml index b8df6d50d..7cb90d96a 100644 --- a/dmp-backend/web/src/main/resources/config/transformer.yml +++ b/dmp-backend/web/src/main/resources/config/transformer.yml @@ -1,18 +1,21 @@ transformer: sources: - url: http://localhost:8084 + transformerId: docx-file-transformer codes: [ docx, pdf ] issuer-url: ${IDP_ISSUER_URI_TOKEN:} client-id: ${IDP_APIKEY_CLIENT_ID:} client-secret: ${IDP_APIKEY_CLIENT_SECRET:} scope: ${IDP_APIKEY_SCOPE:} - url: http://localhost:8086 + transformerId: rda-file-transformer codes: [ json ] issuer-url: ${IDP_ISSUER_URI_TOKEN:} client-id: ${IDP_APIKEY_CLIENT_ID:} client-secret: ${IDP_APIKEY_CLIENT_SECRET:} scope: ${IDP_APIKEY_SCOPE:} - url: http://localhost:8087 + transformerId: import-export-file-transformer codes: [ xml ] issuer-url: ${IDP_ISSUER_URI_TOKEN:} client-id: ${IDP_APIKEY_CLIENT_ID:}