MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

deposit changes

This commit is contained in:
Efstratios Giannopoulos 2024-03-11 15:01:56 +02:00
parent a73eef6529
commit 403e618d04
17 changed files with 334 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java
View File

@ -40,6 +40,7 @@ public class AuditableAction {
public static final EventId Dmp_Invite_Users = new EventId(5008, "Dmp_Invite_Users");
public static final EventId Dmp_Invite_Accept = new EventId(5009, "Dmp_Invite_Accept");
public static final EventId Dmp_PublicQuery = new EventId(5010, "Dmp_PublicQuery");
public static final EventId Dmp_Export = new EventId(5011, "Dmp_Export");
public static final EventId Description_Query = new EventId(6000, "Description_Query");
public static final EventId Description_Lookup = new EventId(6001, "Description_Lookup");

3
dmp-backend/core/src/main/java/eu/eudat/commons/enums/StorageType.java
View File

@ -9,7 +9,8 @@ public enum StorageType implements DatabaseEnum<Short> {
Temp((short) 0),
Main((short) 1),
Transformer((short)2);
Transformer((short)2),
Deposit((short)3);
private final Short value;

18
dmp-backend/core/src/main/java/eu/eudat/configurations/deposit/DepositProperties.java
View File

@ -26,6 +26,8 @@ public class DepositProperties {
private String clientId;
private String clientSecret;
private String scope;
private String pdfTransformerId;
private String rdaTransformerId;
public String getRepositoryId() {
return repositoryId;
@ -74,5 +76,21 @@ public class DepositProperties {
public void setScope(String scope) {
this.scope = scope;
}
public String getPdfTransformerId() {
return pdfTransformerId;
}
public void setPdfTransformerId(String pdfTransformerId) {
this.pdfTransformerId = pdfTransformerId;
}
public String getRdaTransformerId() {
return rdaTransformerId;
}
public void setRdaTransformerId(String rdaTransformerId) {
this.rdaTransformerId = rdaTransformerId;
}
}
}

8
dmp-backend/core/src/main/java/eu/eudat/configurations/transformer/TransformerProperties.java
View File

@ -22,6 +22,7 @@ public class TransformerProperties {
public static class TransformerSource {
private final String url;
private final String transformerId;
private final List<String> codes;
private final String issuerUrl;
private final String clientId;
@ -29,8 +30,9 @@ public class TransformerProperties {
private final String scope;
@ConstructorBinding
public TransformerSource(String url, List<String> codes, String issuerUrl, String clientId, String clientSecret, String scope) {
public TransformerSource(String url, String transformerId, List<String> codes, String issuerUrl, String clientId, String clientSecret, String scope) {
this.url = url;
this.transformerId = transformerId;
this.codes = codes;
this.issuerUrl = issuerUrl;
this.clientId = clientId;
@ -61,5 +63,9 @@ public class TransformerProperties {
public List<String> getCodes() {
return codes;
}
public String getTransformerId() {
return transformerId;
}
}
}

7
dmp-backend/core/src/main/java/eu/eudat/model/builder/commonmodels/dmp/DmpCommonModelBuilder.java
View File

@ -49,7 +49,6 @@ public class DmpCommonModelBuilder extends BaseCommonModelBuilder<DmpModel, DmpE
private final JsonHandlingService jsonHandlingService;
private FileEnvelopeModel pdfFile;
private FileEnvelopeModel rdaJsonFile;
private FileEnvelopeModel supportingFilesZip;
private String repositoryId;
private boolean disableDescriptions;
private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
@ -79,11 +78,6 @@ public class DmpCommonModelBuilder extends BaseCommonModelBuilder<DmpModel, DmpE
return this;
}
public DmpCommonModelBuilder setSupportingFilesZip(FileEnvelopeModel supportingFilesZip) {
this.supportingFilesZip = supportingFilesZip;
return this;
}
public DmpCommonModelBuilder setRepositoryId(String repositoryId) {
this.repositoryId = repositoryId;
return this;
@ -134,7 +128,6 @@ public class DmpCommonModelBuilder extends BaseCommonModelBuilder<DmpModel, DmpE
m.setUpdatedAt(d.getUpdatedAt());
m.setPdfFile(this.pdfFile);
m.setRdaJsonFile(this.rdaJsonFile);
m.setSupportingFilesZip(this.supportingFilesZip);
if (d.getVersion() > (short)1) m.setPreviousDOI(this.getPreviousDOI(d.getGroupId(), d.getId()));
switch (d.getAccessType()){
case Public -> m.setAccessType(DmpAccessType.Public);

9
dmp-backend/core/src/main/java/eu/eudat/model/file/ExportRequestModel.java
View File

@ -5,6 +5,7 @@ import java.util.UUID;
public class ExportRequestModel {
private UUID id;
private String repositoryId;
private String format;
public UUID getId() {
@ -22,4 +23,12 @@ public class ExportRequestModel {
public void setFormat(String format) {
this.format = format;
}
public String getRepositoryId() {
return repositoryId;
}
public void setRepositoryId(String repositoryId) {
this.repositoryId = repositoryId;
}
}

1
dmp-backend/core/src/main/java/eu/eudat/query/UserQuery.java
View File

@ -150,6 +150,7 @@ public class UserQuery extends QueryBase<UserEntity> {
if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseUser)) return null;
UUID userId;
if (this.authorize.contains(AuthorizationFlags.Owner)) userId = this.userScope.getUserIdSafe();
if (this.authorize.contains(AuthorizationFlags.Public)) userId = this.userScope.getUserIdSafe();
else userId = null;
List<Predicate> predicates = new ArrayList<>();

110
dmp-backend/core/src/main/java/eu/eudat/service/deposit/DepositServiceImpl.java
View File

@ -3,19 +3,34 @@ package eu.eudat.service.deposit;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.commonmodels.models.FileEnvelopeModel;
import eu.eudat.commonmodels.models.description.DescriptionModel;
import eu.eudat.commonmodels.models.description.FieldModel;
import eu.eudat.commonmodels.models.description.PropertyDefinitionFieldSetItemModel;
import eu.eudat.commonmodels.models.description.PropertyDefinitionFieldSetModel;
import eu.eudat.commonmodels.models.dmp.DmpModel;
import eu.eudat.commons.enums.StorageType;
import eu.eudat.commons.scope.user.UserScope;
import eu.eudat.configurations.deposit.DepositProperties;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.DescriptionTemplateEntity;
import eu.eudat.data.DmpEntity;
import eu.eudat.depositinterface.repository.DepositClient;
import eu.eudat.depositinterface.repository.DepositConfiguration;
import eu.eudat.model.DescriptionTemplate;
import eu.eudat.model.EntityDoi;
import eu.eudat.model.StorageFile;
import eu.eudat.model.builder.commonmodels.DepositConfigurationBuilder;
import eu.eudat.model.builder.commonmodels.dmp.DmpCommonModelBuilder;
import eu.eudat.model.persist.StorageFilePersist;
import eu.eudat.model.persist.deposit.DepositAuthenticateRequest;
import eu.eudat.model.persist.deposit.DepositRequest;
import eu.eudat.model.persist.EntityDoiPersist;
import eu.eudat.query.DescriptionTemplateQuery;
import eu.eudat.query.DmpQuery;
import eu.eudat.service.entitydoi.EntityDoiService;
import eu.eudat.service.storage.StorageFileProperties;
import eu.eudat.service.storage.StorageFileService;
import eu.eudat.service.transformer.FileTransformerService;
import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeCacheService;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeFilterFunction;
@ -23,7 +38,10 @@ import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeModel;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyNotFoundException;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.fieldset.FieldSet;
import gr.cite.tools.validation.ValidatorFactory;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -32,11 +50,12 @@ import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.net.URLConnection;
import java.time.Duration;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class DepositServiceImpl implements DepositService {
@ -52,7 +71,12 @@ public class DepositServiceImpl implements DepositService {
private final MessageSource messageSource;
private final BuilderFactory builderFactory;
private final DepositConfigurationCacheService depositConfigurationCacheService;
private final FileTransformerService fileTransformerService;
private final StorageFileService storageFileService;
private final UserScope userScope;
private final ValidatorFactory validatorFactory;
private final StorageFileProperties storageFileProperties;
private final ConventionService conventionService;
@Autowired
public DepositServiceImpl(DepositProperties depositProperties,
TokenExchangeCacheService tokenExchangeCacheService,
@ -61,7 +85,7 @@ public class DepositServiceImpl implements DepositService {
EntityDoiService doiService,
QueryFactory queryFactory,
MessageSource messageSource,
BuilderFactory builderFactory, DepositConfigurationCacheService depositConfigurationCacheService) {
BuilderFactory builderFactory, DepositConfigurationCacheService depositConfigurationCacheService, FileTransformerService fileTransformerService, StorageFileService storageFileService, UserScope userScope, ValidatorFactory validatorFactory, StorageFileProperties storageFileProperties, ConventionService conventionService) {
this.depositProperties = depositProperties;
this.tokenExchangeCacheService = tokenExchangeCacheService;
this.authorizationService = authorizationService;
@ -71,6 +95,12 @@ public class DepositServiceImpl implements DepositService {
this.messageSource = messageSource;
this.builderFactory = builderFactory;
this.depositConfigurationCacheService = depositConfigurationCacheService;
this.fileTransformerService = fileTransformerService;
this.storageFileService = storageFileService;
this.userScope = userScope;
this.validatorFactory = validatorFactory;
this.storageFileProperties = storageFileProperties;
this.conventionService = conventionService;
this.clients = new HashMap<>();
}
@ -130,29 +160,48 @@ public class DepositServiceImpl implements DepositService {
//GK: Forth make the required files to be uploaded with the deposit
//TODO: Properly create required files
FileEnvelopeModel docEnvelope = new FileEnvelopeModel();
DepositProperties.DepositSource source = depositProperties.getSources().stream().filter(depositSource -> depositSource.getRepositoryId().equals(dmpDepositModel.getRepositoryId())).findFirst().orElse(null);
if (source == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{dmpDepositModel.getRepositoryId(), DepositProperties.DepositSource.class.getSimpleName()}, LocaleContextHolder.getLocale()));
eu.eudat.model.file.FileEnvelope pdfFile = this.fileTransformerService.exportDmp(dmpEntity.getId(), source.getPdfTransformerId(),"pdf");
eu.eudat.model.file.FileEnvelope rda = this.fileTransformerService.exportDmp(dmpEntity.getId(), source.getRdaTransformerId(),"json");
FileEnvelopeModel pdfEnvelope = new FileEnvelopeModel();
FileEnvelopeModel jsonEnvelope = new FileEnvelopeModel();
FileEnvelopeModel zip = new FileEnvelopeModel();
//TODO: Create Files
// File zip = new File(environment.getProperty("temp.temp") + UUID.randomUUID() + ".zip");
// try {
// docEnvelope.setFilename("test.docx");
// docEnvelope.setFile(this.storageFileService.getH2020TemplateFile());
// File pdfFile = PDFUtils.convertToPDF(docEnvelope, environment);
// pdfEnvelope.setFilename("test.pdf");
// pdfEnvelope.setFile(pdfFile);
// File jsonFile = new File(this.environment.getProperty("temp.temp") + UUID.randomUUID() + ".json");
// jsonEnvelope.setFilename("test.json");
// jsonEnvelope.setFile(jsonFile);
//
// } catch (IOException e) {
// logger.error(e.getMessage(), e);
// }
pdfEnvelope.setFilename(pdfFile.getFilename());
jsonEnvelope.setFilename(rda.getFilename());
if (!depositClient.getConfiguration().isUseSharedStorage()){
pdfEnvelope.setFile(pdfFile.getFile());
jsonEnvelope.setFile(rda.getFile());
} else {
pdfEnvelope.setFileRef(this.addFileToSharedStorage(pdfFile));
jsonEnvelope.setFileRef(this.addFileToSharedStorage(rda));
}
//GK: Fifth Transform them to the DepositModel
DmpModel depositModel = this.builderFactory.builder(DmpCommonModelBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic)
.setRepositoryId(dmpDepositModel.getRepositoryId()).setPdfFile(pdfEnvelope).setRdaJsonFile(jsonEnvelope).setSupportingFilesZip(zip).build(dmpEntity);
.setRepositoryId(dmpDepositModel.getRepositoryId()).setPdfFile(pdfEnvelope).setRdaJsonFile(jsonEnvelope).build(dmpEntity);
//TODO
// if (depositModel != null && !this.conventionService.isListNullOrEmpty(depositModel.getDescriptions())){
// for (DescriptionModel descriptionModel :depositModel.getDescriptions()){
// DescriptionTemplateEntity descriptionTemplate = this.queryFactory.query(DescriptionTemplateQuery.class).ids(descriptionModel.getDescriptionTemplate().getId()).firstAs(new BaseFieldSet().ensure(DescriptionTemplate._definition));
// if (descriptionTemplate == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{descriptionModel.getDescriptionTemplate().getId(), DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));
//
// if (descriptionModel.getProperties() != null && descriptionModel.getProperties().getFieldSets() != null){
// for(PropertyDefinitionFieldSetItemModel propertyDefinitionFieldSetItemModel : descriptionModel.getProperties().getFieldSets().values().stream().filter(x-> x.getItems() != null).map(PropertyDefinitionFieldSetModel::getItems).flatMap(Collection::stream).toList()){
// if (propertyDefinitionFieldSetItemModel.getFields() != null){
// for(Map.Entry<String, FieldModel> entrySet : propertyDefinitionFieldSetItemModel.getFields().entrySet()){
//
// }
// }
// }
// }
//
// }
//
// }
//GK: Sixth Perform the deposit
String doi = depositClient.deposit(depositModel, dmpDepositModel.getAccessToken());
@ -166,6 +215,19 @@ public class DepositServiceImpl implements DepositService {
return doiService.persist(doiPersist, dmpDepositModel.getProject());
}
private String addFileToSharedStorage(eu.eudat.model.file.FileEnvelope file) throws IOException {
StorageFilePersist storageFilePersist = new StorageFilePersist();
storageFilePersist.setName(FilenameUtils.removeExtension(file.getFilename()));
storageFilePersist.setExtension(FilenameUtils.getExtension(file.getFilename()));
storageFilePersist.setMimeType(URLConnection.guessContentTypeFromName(file.getFilename()));
storageFilePersist.setOwnerId(this.userScope.getUserIdSafe());
storageFilePersist.setStorageType(StorageType.Temp);
storageFilePersist.setLifetime(Duration.ofSeconds(this.storageFileProperties.getTempStoreLifetimeSeconds())); //TODO
this.validatorFactory.validator(StorageFilePersist.StorageFilePersistValidator.class).validateForce(storageFilePersist);
StorageFile persisted = this.storageFileService.persistBytes(storageFilePersist, file.getFile(), new BaseFieldSet(StorageFile._id, StorageFile._fileRef));
return persisted.getFileRef();
}
@Override
public String getLogo(String repositoryId) {
this.authorizationService.authorizeForce(Permission.BrowseDeposit);

2
dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpService.java
View File

@ -34,7 +34,7 @@ public interface DmpService {
List<DmpUser> assignUsers(UUID dmp, List<DmpUserPersist> model, FieldSet fields) throws InvalidApplicationException, IOException;
Dmp removeUser(DmpUserRemovePersist model, FieldSet fields) throws InvalidApplicationException, IOException;
ResponseEntity<byte[]> export(UUID id, String exportType) throws InvalidApplicationException, IOException;
ResponseEntity<byte[]> export(UUID id, String transformerId, String exportType) throws InvalidApplicationException, IOException;
void inviteUsers(UUID id, List<DmpUserPersist> users) throws InvalidApplicationException, JAXBException, IOException;

4
dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpServiceImpl.java
View File

@ -518,10 +518,10 @@ public class DmpServiceImpl implements DmpService {
}
@Override
public ResponseEntity<byte[]> export(UUID id, String exportType) throws InvalidApplicationException, IOException {
public ResponseEntity<byte[]> export(UUID id, String transformerId, String exportType) throws InvalidApplicationException, IOException {
HttpHeaders headers = new HttpHeaders();
FileEnvelope fileEnvelope = this.fileTransformerService.exportDmp(id, exportType);
FileEnvelope fileEnvelope = this.fileTransformerService.exportDmp(id, transformerId, exportType);
headers.add("Content-Disposition", "attachment;filename=" + fileEnvelope.getFilename());
byte[] data = fileEnvelope.getFile();
headers.setContentType(MediaType.APPLICATION_OCTET_STREAM);

175
dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerService.java
View File

@ -1,177 +1,14 @@
package eu.eudat.service.transformer;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.cache.transformer.FileTransformerConfigurationCache;
import eu.eudat.commonmodels.models.FileEnvelopeModel;
import eu.eudat.commonmodels.models.description.DescriptionModel;
import eu.eudat.commonmodels.models.dmp.DmpModel;
import eu.eudat.commons.enums.StorageType;
import eu.eudat.configurations.transformer.TransformerProperties;
import eu.eudat.file.transformer.interfaces.FileTransformerConfiguration;
import eu.eudat.file.transformer.models.misc.FileFormat;
import eu.eudat.model.Description;
import eu.eudat.model.Dmp;
import eu.eudat.model.builder.commonmodels.description.DescriptionCommonModelBuilder;
import eu.eudat.model.builder.commonmodels.dmp.DmpCommonModelBuilder;
import eu.eudat.model.file.TransformerCacheModel;
import eu.eudat.query.DescriptionQuery;
import eu.eudat.query.DmpQuery;
import eu.eudat.repository.TransformerRepository;
import eu.eudat.service.storage.StorageFileService;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeCacheService;
import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeFilterFunction;
import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeModel;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyNotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.core.publisher.Mono;
import javax.management.InvalidApplicationException;
import java.io.IOException;
import java.net.URI;
import java.util.*;
import java.util.List;
import java.util.UUID;
@Service
public class FileTransformerService {
private static final Logger logger = LoggerFactory.getLogger(FileTransformerService.class);
public interface FileTransformerService {
List<FileFormat> getAvailableExportFileFormats();
private final TransformerProperties transformerProperties;
private final Map<String, TransformerRepository> clients;
private final TokenExchangeCacheService tokenExchangeCacheService;
private final FileTransformerConfigurationCache fileTransformerConfigurationCache;
private final AuthorizationService authorizationService;
private final WebClient.Builder webClientBuilder;
private final QueryFactory queryFactory;
private final BuilderFactory builderFactory;
private final StorageFileService storageFileService;
private final MessageSource messageSource;
eu.eudat.model.file.FileEnvelope exportDmp(UUID dmpId, String repositoryId, String format);
@Autowired
public FileTransformerService(TransformerProperties transformerProperties, TokenExchangeCacheService tokenExchangeCacheService, FileTransformerConfigurationCache fileTransformerConfigurationCache, WebClient.Builder builder, AuthorizationService authorizationService,
QueryFactory queryFactory, BuilderFactory builderFactory, StorageFileService storageFileService, MessageSource messageSource) {
this.transformerProperties = transformerProperties;
this.tokenExchangeCacheService = tokenExchangeCacheService;
this.fileTransformerConfigurationCache = fileTransformerConfigurationCache;
this.authorizationService = authorizationService;
this.webClientBuilder = builder;
this.queryFactory = queryFactory;
this.builderFactory = builderFactory;
this.storageFileService = storageFileService;
this.messageSource = messageSource;
this.clients = new HashMap<>();
}
private TransformerRepository getRepository(String repoId) {
if (this.clients.containsKey(repoId)) return this.clients.get(repoId);
//GK: It's register time
TransformerProperties.TransformerSource source = transformerProperties.getSources().stream().filter(depositSource -> depositSource.getCodes().contains(repoId)).findFirst().orElse(null);
if (source != null) {
String host = URI.create(source.getUrl()).getHost();
TokenExchangeModel tokenExchangeModel = new TokenExchangeModel(host + "_" + source.getClientId(), source.getIssuerUrl(), source.getClientId(), source.getClientSecret(), source.getScope());
TokenExchangeFilterFunction tokenExchangeFilterFunction = new TokenExchangeFilterFunction(this.tokenExchangeCacheService, tokenExchangeModel);
TransformerRepository repository = new TransformerRepository(webClientBuilder.baseUrl(source.getUrl() + "/api/file-transformer").filters(exchangeFilterFunctions -> {
exchangeFilterFunctions.add(tokenExchangeFilterFunction);
exchangeFilterFunctions.add(logRequest());
}).build());
source.getCodes().forEach(code -> this.clients.put(code, repository));
return repository;
}
return null;
}
public List<FileFormat> getAvailableExportFileFormats() {
List<FileFormat> formats = new ArrayList<>();
List<FileTransformerConfiguration> configurations = this.getAvailableConfigurations();
if(configurations != null){
for (FileTransformerConfiguration configuration : configurations){
if (configuration != null && configuration.getExportVariants() != null) formats.addAll(configuration.getExportVariants());
}
}
return formats;
}
private List<FileTransformerConfiguration> getAvailableConfigurations() {
TransformerCacheModel configs = fileTransformerConfigurationCache.lookup("base");
if (configs == null) {
List<FileTransformerConfiguration> configurations = new ArrayList<>();
//GK: So much for lazy loading
List<TransformerRepository> repositories = transformerProperties.getSources().stream().map(depositSource -> getRepository(depositSource.getCodes().getFirst())).toList();
repositories = new ArrayList<>(repositories);
repositories.forEach((client) -> {
try {
FileTransformerConfiguration repositoryConfig = client.getConfiguration();
if (repositoryConfig != null) {
configurations.add(repositoryConfig);
}
} catch (Exception e) {
logger.warn(e.getLocalizedMessage(), e);
}
});
configs = new TransformerCacheModel(configurations);
this.fileTransformerConfigurationCache.put("base", configs);
}
return configs.getConfigurations();
}
public eu.eudat.model.file.FileEnvelope exportDmp(UUID dmpId, String format) throws InvalidApplicationException, IOException {
this.authorizationService.authorize(Permission.EditDmp);
//GK: First get the right client
TransformerRepository repository = getRepository(format);
if (repository == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{format, TransformerRepository.class.getSimpleName()}, LocaleContextHolder.getLocale()));
//GK: Second get the Target Data Management Plan
DmpQuery query = this.queryFactory.query(DmpQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(dmpId);
DmpModel dmpFileTransformerModel = this.builderFactory.builder(DmpCommonModelBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(query.first());
if (dmpFileTransformerModel == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{dmpId, Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale()));
FileEnvelopeModel fileEnvelope = repository.exportDmp(dmpFileTransformerModel, format);
eu.eudat.model.file.FileEnvelope result = new eu.eudat.model.file.FileEnvelope();
byte[] data = repository.getConfiguration().isUseSharedStorage() ? storageFileService.readByFileRefAsBytesSafe(fileEnvelope.getFileRef(), StorageType.Transformer) : fileEnvelope.getFile();
result.setFile(data);
result.setFilename(fileEnvelope.getFilename());
return result;
}
public eu.eudat.model.file.FileEnvelope exportDescription(UUID descriptionId, String format) {
this.authorizationService.authorize(Permission.EditDmp);
//GK: First get the right client
TransformerRepository repository = getRepository(format);
if (repository == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{format, TransformerRepository.class.getSimpleName()}, LocaleContextHolder.getLocale()));
//GK: Second get the Target Data Management Plan
DescriptionQuery query = this.queryFactory.query(DescriptionQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(descriptionId);
DescriptionModel descriptionFileTransformerModel = this.builderFactory.builder(DescriptionCommonModelBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(query.first());
if (descriptionFileTransformerModel == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{descriptionId, Description.class.getSimpleName()}, LocaleContextHolder.getLocale()));
FileEnvelopeModel fileEnvelope = repository.exportDescription(descriptionFileTransformerModel, format);
eu.eudat.model.file.FileEnvelope result = new eu.eudat.model.file.FileEnvelope();
byte[] data = repository.getConfiguration().isUseSharedStorage() ? storageFileService.readByFileRefAsBytesSafe(fileEnvelope.getFileRef(), StorageType.Transformer) : fileEnvelope.getFile(); //TODO: shared storage should be per repository
result.setFile(data);
result.setFilename(fileEnvelope.getFilename());
return result;
}
// This method returns filter function which will log request data
private static ExchangeFilterFunction logRequest() {
return ExchangeFilterFunction.ofRequestProcessor(clientRequest -> {
logger.info("Request: {} {}", clientRequest.method(), clientRequest.url());
clientRequest.headers().forEach((name, values) -> values.forEach(value -> logger.info("{}={}", name, value)));
return Mono.just(clientRequest);
});
}
eu.eudat.model.file.FileEnvelope exportDescription(UUID descriptionId, String format);
}

178
dmp-backend/core/src/main/java/eu/eudat/service/transformer/FileTransformerServiceImpl.java Normal file
View File

@ -0,0 +1,178 @@
package eu.eudat.service.transformer;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.cache.transformer.FileTransformerConfigurationCache;
import eu.eudat.commonmodels.models.FileEnvelopeModel;
import eu.eudat.commonmodels.models.description.DescriptionModel;
import eu.eudat.commonmodels.models.dmp.DmpModel;
import eu.eudat.commons.enums.StorageType;
import eu.eudat.configurations.transformer.TransformerProperties;
import eu.eudat.file.transformer.interfaces.FileTransformerConfiguration;
import eu.eudat.file.transformer.models.misc.FileFormat;
import eu.eudat.model.Description;
import eu.eudat.model.Dmp;
import eu.eudat.model.builder.commonmodels.description.DescriptionCommonModelBuilder;
import eu.eudat.model.builder.commonmodels.dmp.DmpCommonModelBuilder;
import eu.eudat.model.file.TransformerCacheModel;
import eu.eudat.query.DescriptionQuery;
import eu.eudat.query.DmpQuery;
import eu.eudat.repository.TransformerRepository;
import eu.eudat.service.storage.StorageFileService;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeCacheService;
import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeFilterFunction;
import gr.cite.commons.web.oidc.filter.webflux.TokenExchangeModel;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyNotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.core.publisher.Mono;
import java.net.URI;
import java.util.*;
@Service
public class FileTransformerServiceImpl implements FileTransformerService {
private static final Logger logger = LoggerFactory.getLogger(FileTransformerServiceImpl.class);
private final TransformerProperties transformerProperties;
private final Map<String, TransformerRepository> clients;
private final TokenExchangeCacheService tokenExchangeCacheService;
private final FileTransformerConfigurationCache fileTransformerConfigurationCache;
private final AuthorizationService authorizationService;
private final WebClient.Builder webClientBuilder;
private final QueryFactory queryFactory;
private final BuilderFactory builderFactory;
private final StorageFileService storageFileService;
private final MessageSource messageSource;
@Autowired
public FileTransformerServiceImpl(TransformerProperties transformerProperties, TokenExchangeCacheService tokenExchangeCacheService, FileTransformerConfigurationCache fileTransformerConfigurationCache, WebClient.Builder builder, AuthorizationService authorizationService,
QueryFactory queryFactory, BuilderFactory builderFactory, StorageFileService storageFileService, MessageSource messageSource) {
this.transformerProperties = transformerProperties;
this.tokenExchangeCacheService = tokenExchangeCacheService;
this.fileTransformerConfigurationCache = fileTransformerConfigurationCache;
this.authorizationService = authorizationService;
this.webClientBuilder = builder;
this.queryFactory = queryFactory;
this.builderFactory = builderFactory;
this.storageFileService = storageFileService;
this.messageSource = messageSource;
this.clients = new HashMap<>();
}
private TransformerRepository getRepository(String repoId) {
if (this.clients.containsKey(repoId)) return this.clients.get(repoId);
//GK: It's register time
TransformerProperties.TransformerSource source = transformerProperties.getSources().stream().filter(depositSource -> depositSource.getCodes().contains(repoId)).findFirst().orElse(null);
if (source != null) {
String host = URI.create(source.getUrl()).getHost();
TokenExchangeModel tokenExchangeModel = new TokenExchangeModel(host + "_" + source.getClientId(), source.getIssuerUrl(), source.getClientId(), source.getClientSecret(), source.getScope());
TokenExchangeFilterFunction tokenExchangeFilterFunction = new TokenExchangeFilterFunction(this.tokenExchangeCacheService, tokenExchangeModel);
TransformerRepository repository = new TransformerRepository(webClientBuilder.baseUrl(source.getUrl() + "/api/file-transformer").filters(exchangeFilterFunctions -> {
exchangeFilterFunctions.add(tokenExchangeFilterFunction);
exchangeFilterFunctions.add(logRequest());
}).build());
source.getCodes().forEach(code -> this.clients.put(code, repository));
return repository;
}
return null;
}
@Override
public List<FileFormat> getAvailableExportFileFormats() {
List<FileFormat> formats = new ArrayList<>();
List<FileTransformerConfiguration> configurations = this.getAvailableConfigurations();
if(configurations != null){
for (FileTransformerConfiguration configuration : configurations){
if (configuration != null && configuration.getExportVariants() != null) formats.addAll(configuration.getExportVariants());
}
}
return formats;
}
private List<FileTransformerConfiguration> getAvailableConfigurations() {
TransformerCacheModel configs = fileTransformerConfigurationCache.lookup("base");
if (configs == null) {
List<FileTransformerConfiguration> configurations = new ArrayList<>();
//GK: So much for lazy loading
List<TransformerRepository> repositories = transformerProperties.getSources().stream().map(depositSource -> getRepository(depositSource.getCodes().getFirst())).toList();
repositories = new ArrayList<>(repositories);
repositories.forEach((client) -> {
try {
FileTransformerConfiguration repositoryConfig = client.getConfiguration();
if (repositoryConfig != null) {
configurations.add(repositoryConfig);
}
} catch (Exception e) {
logger.warn(e.getLocalizedMessage(), e);
}
});
configs = new TransformerCacheModel(configurations);
this.fileTransformerConfigurationCache.put("base", configs);
}
return configs.getConfigurations();
}
@Override
public eu.eudat.model.file.FileEnvelope exportDmp(UUID dmpId, String repositoryId, String format) {
this.authorizationService.authorize(Permission.EditDmp);
//GK: First get the right client
TransformerRepository repository = getRepository(repositoryId);
if (repository == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{repositoryId, TransformerRepository.class.getSimpleName()}, LocaleContextHolder.getLocale()));
//GK: Second get the Target Data Management Plan
DmpQuery query = this.queryFactory.query(DmpQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(dmpId);
DmpModel dmpFileTransformerModel = this.builderFactory.builder(DmpCommonModelBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(query.first());
if (dmpFileTransformerModel == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{dmpId, Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale()));
FileEnvelopeModel fileEnvelope = repository.exportDmp(dmpFileTransformerModel, format);
eu.eudat.model.file.FileEnvelope result = new eu.eudat.model.file.FileEnvelope();
byte[] data = repository.getConfiguration().isUseSharedStorage() ? storageFileService.readByFileRefAsBytesSafe(fileEnvelope.getFileRef(), StorageType.Transformer) : fileEnvelope.getFile();
result.setFile(data);
result.setFilename(fileEnvelope.getFilename());
return result;
}
@Override
public eu.eudat.model.file.FileEnvelope exportDescription(UUID descriptionId, String format) {
this.authorizationService.authorize(Permission.EditDmp);
//GK: First get the right client
TransformerRepository repository = getRepository(format);
if (repository == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{format, TransformerRepository.class.getSimpleName()}, LocaleContextHolder.getLocale()));
//GK: Second get the Target Data Management Plan
DescriptionQuery query = this.queryFactory.query(DescriptionQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(descriptionId);
DescriptionModel descriptionFileTransformerModel = this.builderFactory.builder(DescriptionCommonModelBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(query.first());
if (descriptionFileTransformerModel == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{descriptionId, Description.class.getSimpleName()}, LocaleContextHolder.getLocale()));
FileEnvelopeModel fileEnvelope = repository.exportDescription(descriptionFileTransformerModel, format);
eu.eudat.model.file.FileEnvelope result = new eu.eudat.model.file.FileEnvelope();
byte[] data = repository.getConfiguration().isUseSharedStorage() ? storageFileService.readByFileRefAsBytesSafe(fileEnvelope.getFileRef(), StorageType.Transformer) : fileEnvelope.getFile(); //TODO: shared storage should be per repository
result.setFile(data);
result.setFilename(fileEnvelope.getFilename());
return result;
}
// This method returns filter function which will log request data
private static ExchangeFilterFunction logRequest() {
return ExchangeFilterFunction.ofRequestProcessor(clientRequest -> {
logger.info("Request: {} {}", clientRequest.method(), clientRequest.url());
clientRequest.headers().forEach((name, values) -> values.forEach(value -> logger.info("{}={}", name, value)));
return Mono.just(clientRequest);
});
}
}

14
dmp-backend/web/src/main/java/eu/eudat/controllers/DmpController.java
View File

@ -215,11 +215,17 @@ public class DmpController {
return new QueryResult<>(persisted);
}
@GetMapping("{id}/export/{type}")
public ResponseEntity<byte[]> export(@PathVariable("id") UUID id, @PathVariable("type") String exportType) throws InvalidApplicationException, IOException {
logger.debug(new MapLogEntry("exporting dmp"));
@GetMapping("{id}/export/{transformerId}/{type}")
public ResponseEntity<byte[]> export(@PathVariable("id") UUID id, @PathVariable("transformerId") String transformerId, @PathVariable("type") String exportType) throws InvalidApplicationException, IOException {
logger.debug(new MapLogEntry("exporting dmp").And("id", id).And("transformerId", transformerId).And("exportType", exportType));
return this.dmpService.export(id, exportType);
ResponseEntity<byte[]> bytes = this.dmpService.export(id, transformerId, exportType);
this.auditService.track(AuditableAction.Dmp_Export, Map.ofEntries(
new AbstractMap.SimpleEntry<String, Object>("id", id),
new AbstractMap.SimpleEntry<String, Object>("transformerId", transformerId),
new AbstractMap.SimpleEntry<String, Object>("exportType", exportType)
));
return bytes;
}
@PostMapping("{id}/invite-users")

2
dmp-backend/web/src/main/java/eu/eudat/controllers/FileTransformerController.java
View File

@ -52,7 +52,7 @@ public class FileTransformerController {
logger.debug(new MapLogEntry("exporting dmp"));
HttpHeaders headers = new HttpHeaders();
FileEnvelope fileEnvelope = this.fileTransformerService.exportDmp(requestModel.getId(), requestModel.getFormat());
FileEnvelope fileEnvelope = this.fileTransformerService.exportDmp(requestModel.getId(), requestModel.getRepositoryId(), requestModel.getFormat());
headers.add("Content-Disposition", "attachment;filename=" + fileEnvelope.getFilename());
byte[] data = fileEnvelope.getFile();
headers.setContentType(MediaType.APPLICATION_OCTET_STREAM);

2
dmp-backend/web/src/main/resources/config/deposit.yml
View File

@ -2,6 +2,8 @@ deposit:
sources:
- url: http://localhost:8082
repositoryId: zenodo
pdfTransformerId: docx-file-transformer
rdaTransformerId: rda-file-transformer
issuer-url: ${ZENODO_ISSUER_URI:IDP_APIKEY_ISSUER_URI}/protocol/openid-connect/token
client-id: ${ZENODO_DEPOSIT_CLIENT_ID:}
client-secret: ${ZENODO_DEPOSIT_CLIENT_SECRET:}

8
dmp-backend/web/src/main/resources/config/storage-devel.yml
View File

@ -3,11 +3,13 @@ storage:
defaultLanguage: en
storages:
- type: Temp
basePath: ./storage/temp
basePath: ${FILE_STORAGE}/temp
- type: Main
basePath: ./storage/main
basePath: ${FILE_STORAGE}/main
- type: Transformer
basePath: ${TRANSFORMER_BASE_PATH}
basePath: ${FILE_STORAGE}/transformer
- type: Deposit
basePath: ${FILE_STORAGE}/deposit
static-files:
externalUrls: dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml
semantics: dmp-backend/web/src/main/resources/Semantics.json

3
dmp-backend/web/src/main/resources/config/transformer.yml
View File

@ -1,18 +1,21 @@
transformer:
sources:
- url: http://localhost:8084
transformerId: docx-file-transformer
codes: [ docx, pdf ]
issuer-url: ${IDP_ISSUER_URI_TOKEN:}
client-id: ${IDP_APIKEY_CLIENT_ID:}
client-secret: ${IDP_APIKEY_CLIENT_SECRET:}
scope: ${IDP_APIKEY_SCOPE:}
- url: http://localhost:8086
transformerId: rda-file-transformer
codes: [ json ]
issuer-url: ${IDP_ISSUER_URI_TOKEN:}
client-id: ${IDP_APIKEY_CLIENT_ID:}
client-secret: ${IDP_APIKEY_CLIENT_SECRET:}
scope: ${IDP_APIKEY_SCOPE:}
- url: http://localhost:8087
transformerId: import-export-file-transformer
codes: [ xml ]
issuer-url: ${IDP_ISSUER_URI_TOKEN:}
client-id: ${IDP_APIKEY_CLIENT_ID:}