migration changes
This commit is contained in:
parent
6ebbb885fb
commit
a28323dcaf
|
|
@ -68,8 +68,7 @@ public enum FieldType implements DatabaseEnum<String> {
|
|||
public static boolean isTextType(FieldType fieldType){
|
||||
return fieldType.equals(FieldType.FREE_TEXT) || fieldType.equals(FieldType.CHECK_BOX) || fieldType.equals(FieldType.TEXT_AREA) ||
|
||||
fieldType.equals(FieldType.RICH_TEXT_AREA) || fieldType.equals(FieldType.UPLOAD) || fieldType.equals(FieldType.BOOLEAN_DECISION) ||
|
||||
fieldType.equals(FieldType.RADIO_BOX) || fieldType.equals(FieldType.CURRENCY) || fieldType.equals(FieldType.SELECT) ||
|
||||
fieldType.equals(FieldType.INTERNAL_ENTRIES_DMPS) || fieldType.equals(FieldType.INTERNAL_ENTRIES_DESCRIPTIONS);
|
||||
fieldType.equals(FieldType.RADIO_BOX) || fieldType.equals(FieldType.CURRENCY) || fieldType.equals(FieldType.SELECT);
|
||||
}
|
||||
|
||||
public static boolean isTextListType(FieldType fieldType){
|
||||
|
|
|
|||
|
|
@ -1,32 +1,22 @@
|
|||
package eu.old.eudat.migration;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import eu.eudat.commons.JsonHandlingService;
|
||||
import eu.eudat.commons.XmlHandlingService;
|
||||
import eu.eudat.commons.enums.*;
|
||||
import eu.eudat.commons.types.description.FieldEntity;
|
||||
import eu.eudat.commons.types.description.PropertyDefinitionEntity;
|
||||
import eu.eudat.commons.types.description.PropertyDefinitionFieldSetEntity;
|
||||
import eu.eudat.commons.enums.DescriptionStatus;
|
||||
import eu.eudat.commons.enums.IsActive;
|
||||
import eu.eudat.commons.types.description.*;
|
||||
import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity;
|
||||
import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity;
|
||||
import eu.eudat.commons.types.dmp.DmpContactEntity;
|
||||
import eu.eudat.commons.types.dmp.DmpPropertiesEntity;
|
||||
import eu.eudat.commons.types.dmpblueprint.DefinitionEntity;
|
||||
import eu.eudat.convention.ConventionService;
|
||||
import eu.eudat.data.*;
|
||||
import eu.eudat.model.Dmp;
|
||||
import eu.eudat.model.DmpDescriptionTemplate;
|
||||
import eu.eudat.query.DescriptionTemplateQuery;
|
||||
import eu.eudat.query.DmpBlueprintQuery;
|
||||
import eu.eudat.query.DmpDescriptionTemplateQuery;
|
||||
import eu.eudat.query.DmpQuery;
|
||||
import eu.old.eudat.data.dao.entities.DatasetDao;
|
||||
import eu.old.eudat.data.entities.DMP;
|
||||
import eu.old.eudat.data.entities.DMPDatasetProfile;
|
||||
import eu.old.eudat.data.entities.Dataset;
|
||||
import eu.old.eudat.logic.services.operations.DatabaseRepository;
|
||||
import eu.old.eudat.models.data.datasetwizard.DatasetWizardModel;
|
||||
import eu.old.eudat.models.data.dmp.DataManagementPlan;
|
||||
import eu.old.eudat.queryable.QueryableList;
|
||||
import gr.cite.tools.data.query.QueryFactory;
|
||||
import gr.cite.tools.exception.MyApplicationException;
|
||||
import gr.cite.tools.fieldset.BaseFieldSet;
|
||||
|
|
@ -41,6 +31,9 @@ import org.xml.sax.SAXException;
|
|||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import java.io.IOException;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
|
@ -55,13 +48,15 @@ public class DatasetMigrationService {
|
|||
private static final int PageSize = 500;
|
||||
private static final boolean TestMode = false;
|
||||
private final EntityManager entityManager;
|
||||
private final ConventionService conventionService;
|
||||
|
||||
public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager) {
|
||||
public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager, ConventionService conventionService) {
|
||||
this.databaseRepository = databaseRepository;
|
||||
this.jsonHandlingService = jsonHandlingService;
|
||||
this.queryFactory = queryFactory;
|
||||
this.xmlHandlingService = xmlHandlingService;
|
||||
this.entityManager = entityManager;
|
||||
this.conventionService = conventionService;
|
||||
}
|
||||
|
||||
public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
|
||||
|
|
@ -92,11 +87,6 @@ public class DatasetMigrationService {
|
|||
descriptionTemplateDefinitionMap.put(descriptionTemplateEntity.getId(), this.xmlHandlingService.fromXml(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition()));
|
||||
}
|
||||
for (Dataset item : items) {
|
||||
//entityManager.detach(item);
|
||||
|
||||
//DatasetWizardModel model = new DatasetWizardModel();
|
||||
//model.fromDataModel(item);
|
||||
|
||||
DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null);
|
||||
|
||||
if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) {
|
||||
|
|
@ -141,14 +131,17 @@ public class DatasetMigrationService {
|
|||
}
|
||||
|
||||
eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity = descriptionTemplateDefinitionMap.getOrDefault(item.getProfile().getId(), null);
|
||||
data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity)));
|
||||
List<Tag> existingTags = new ArrayList<>();
|
||||
|
||||
data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, existingTags)));
|
||||
|
||||
if (data.getCreatedById() == null){
|
||||
logger.error("Migration skipped creator not found " + item.getId());
|
||||
throw new MyApplicationException("Migration skipped creator not found " + item.getId());
|
||||
}
|
||||
|
||||
this.persistTags(existingTags, data);
|
||||
|
||||
this.entityManager.persist(data);
|
||||
this.entityManager.flush();
|
||||
}
|
||||
|
|
@ -156,13 +149,27 @@ public class DatasetMigrationService {
|
|||
page++;
|
||||
}
|
||||
} while (items != null && !items.isEmpty() && !TestMode);
|
||||
throw new MyApplicationException("");
|
||||
}
|
||||
|
||||
private void persistTags(List<Tag> existingTags, DescriptionEntity data){
|
||||
for (Tag tag : existingTags) {
|
||||
TagEntity tagEntity = new TagEntity();
|
||||
tagEntity.setId(UUID.fromString(tag.getId()));
|
||||
tagEntity.setLabel(tag.getName());
|
||||
tagEntity.setCreatedAt(Instant.now());
|
||||
tagEntity.setUpdatedAt(Instant.now());
|
||||
tagEntity.setCreatedById(data.getCreatedById());
|
||||
tagEntity.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(tagEntity);
|
||||
}
|
||||
}
|
||||
|
||||
private List<DmpDescriptionTemplateEntity> getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities){
|
||||
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
|
||||
if (itemDescriptionTemplates.isEmpty()) {
|
||||
logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
|
||||
if (dmpDescriptionTemplateEntities.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) {
|
||||
if (dmpDescriptionTemplateEntities.stream().anyMatch(x -> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId))) {
|
||||
DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity();
|
||||
dmpDescriptionTemplateEntity.setId(UUID.randomUUID());
|
||||
dmpDescriptionTemplateEntity.setDescriptionTemplateGroupId(item.getProfile().getGroupId());
|
||||
|
|
@ -180,43 +187,314 @@ public class DatasetMigrationService {
|
|||
return itemDescriptionTemplates;
|
||||
}
|
||||
|
||||
private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity) {
|
||||
if (item.getProperties() != null) return null;
|
||||
private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, List<Tag> existingTags) {
|
||||
if (this.conventionService.isNullOrEmpty(item.getProperties())) return null;
|
||||
JSONObject jObject = new JSONObject(item.getProperties());
|
||||
Map<String, Object> properties = jObject.toMap();
|
||||
|
||||
PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity();
|
||||
propertyDefinitionEntity.setFieldSets(new HashMap<>());
|
||||
|
||||
List<FieldSetEntity> fieldSetEntities = descriptionTemplateDefinitionEntity.getAllFieldSets();
|
||||
|
||||
// for (String key : properties.keySet()) {
|
||||
// FieldSetEntity currentFieldSet = null;
|
||||
// eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = null;
|
||||
// for (FieldSetEntity fieldSetEntity : fieldSetEntities){
|
||||
// List<eu.eudat.commons.types.descriptiontemplate.FieldEntity> fieldEntities = fieldSetEntity.getFieldById(key);
|
||||
// if (!fieldEntities.isEmpty()){
|
||||
// currentFieldSet = fieldSetEntity;
|
||||
// currentField = fieldEntities.getFirst();
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if (currentFieldSet == null || currentField == null) throw new MyApplicationException("aaa");
|
||||
//
|
||||
// if (!propertyDefinitionEntity.getFieldSets().containsKey(currentFieldSet.getId())){
|
||||
// PropertyDefinitionFieldSetEntity fieldSetEntity = new PropertyDefinitionFieldSetEntity();
|
||||
// fieldSetEntity.setItems(new ArrayList<>());
|
||||
// propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), fieldSetEntity);
|
||||
// }
|
||||
// propertyDefinitionEntity.getFieldSets().get(currentFieldSet.getId())
|
||||
//
|
||||
// FieldEntity fieldEntity = new FieldEntity();
|
||||
// fieldEntity.setKey(key);
|
||||
// fieldEntity.setValue(properties.get(key) != null ? properties.get(key).toString() : null);
|
||||
// fieldEntities.add(fieldEntity);
|
||||
// propertyDefinitionEntity.getFieldSets().put(key, fieldEntity);
|
||||
// }
|
||||
// propertyDefinitionEntity.setFields(fieldEntities);
|
||||
return propertyDefinitionEntity;
|
||||
for (String key : properties.keySet()) {
|
||||
if (key.toLowerCase(Locale.ROOT).trim().startsWith("commentfieldvalue")){
|
||||
if (properties.get(key) != null && this.conventionService.isNullOrEmpty(properties.get(key).toString())) { continue; }
|
||||
FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().equals(key.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null);
|
||||
|
||||
if (currentFieldSet == null) {
|
||||
logger.error("Field set " + key + " not found for comment " + properties.get(key).toString());
|
||||
continue;
|
||||
}
|
||||
|
||||
this.addSimpleCommentField(propertyDefinitionEntity, currentFieldSet, properties.get(key).toString());
|
||||
} else {
|
||||
if (!key.toLowerCase(Locale.ROOT).trim().startsWith("multiple_")) {
|
||||
FieldSetEntity currentFieldSet = null;
|
||||
eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = null;
|
||||
for (FieldSetEntity fieldSetEntity : fieldSetEntities) {
|
||||
List<eu.eudat.commons.types.descriptiontemplate.FieldEntity> fieldEntities = fieldSetEntity.getFieldById(key.trim());
|
||||
if (!fieldEntities.isEmpty()) {
|
||||
currentFieldSet = fieldSetEntity;
|
||||
currentField = fieldEntities.getFirst();
|
||||
}
|
||||
}
|
||||
|
||||
if (currentField == null) {
|
||||
logger.error("Field set not found for field " + key + " " + properties.get(key).toString());
|
||||
continue;
|
||||
}
|
||||
|
||||
this.addSimpleField(propertyDefinitionEntity, currentFieldSet, currentField, properties, existingTags);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Integer> groupOrdinalMapping = new HashMap<>();
|
||||
for (String key : properties.keySet()) {
|
||||
if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) {
|
||||
String newKey = key.trim().substring("multiple_".length());
|
||||
String[] keyParts = newKey.split("_", 4);
|
||||
String fieldSetId = "";
|
||||
String groupId = "";
|
||||
int ordinal = 0;
|
||||
String fieldId = "";
|
||||
try {
|
||||
if (keyParts.length == 4) {
|
||||
fieldSetId = keyParts[0].trim();
|
||||
groupId = keyParts[1].trim();
|
||||
ordinal = Integer.parseInt(keyParts[2].trim());
|
||||
fieldId = keyParts[3].trim();
|
||||
} else if (keyParts.length == 3) {
|
||||
fieldSetId = keyParts[0].trim();
|
||||
groupId = keyParts[1].trim();
|
||||
fieldId = keyParts[2].trim();
|
||||
} else {
|
||||
logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString());
|
||||
continue;
|
||||
//throw new MyApplicationException("Invalid multiple key " + key);
|
||||
}
|
||||
} catch (Exception e){
|
||||
logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = descriptionTemplateDefinitionEntity.getFieldById(fieldId).stream().findFirst().orElse(null);
|
||||
if (currentField == null) {
|
||||
logger.error("Field set not found for field " + key + " " + properties.get(key).toString());
|
||||
continue;
|
||||
}
|
||||
|
||||
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(fieldSetId, null);
|
||||
if (ordinal == 0) {
|
||||
if (propertyDefinitionFieldSetEntity != null ) {
|
||||
if (groupOrdinalMapping.containsKey(groupId + fieldSetId)){
|
||||
ordinal = groupOrdinalMapping.get(groupId + fieldSetId);
|
||||
} else {
|
||||
if (propertyDefinitionFieldSetEntity.getItems().stream().anyMatch(x-> x.getOrdinal() == 0 && !x.getFields().isEmpty())) {
|
||||
ordinal = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> !x.getFields().isEmpty()).map(PropertyDefinitionFieldSetItemEntity::getOrdinal).max(Comparator.comparing(x -> x)).orElse(0) + 1;
|
||||
int finalOrdinal = ordinal;
|
||||
if (propertyDefinitionFieldSetEntity.getItems().stream().anyMatch(x -> x.getOrdinal() == finalOrdinal && !x.getFields().isEmpty()))
|
||||
throw new MyApplicationException("Invalid multiple key " + key);
|
||||
logger.error("Ordinal of group set to " + ordinal + " " + key);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
|
||||
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
|
||||
}
|
||||
}
|
||||
if (!groupOrdinalMapping.containsKey(groupId + fieldSetId)){
|
||||
groupOrdinalMapping.put(groupId + fieldSetId, ordinal);
|
||||
} else {
|
||||
if (groupOrdinalMapping.get(groupId + fieldSetId) != ordinal) throw new MyApplicationException("Invalid multiple key ordinal " + key);
|
||||
}
|
||||
if (propertyDefinitionFieldSetEntity == null) throw new MyApplicationException("Invalid multiple key group " + key);
|
||||
|
||||
this.addMultipleField(propertyDefinitionFieldSetEntity, ordinal, currentField, properties, existingTags);
|
||||
}
|
||||
}
|
||||
return propertyDefinitionEntity;
|
||||
}
|
||||
|
||||
private void addMultipleField(PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, List<Tag> existingTags){
|
||||
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null);
|
||||
|
||||
if (propertyDefinitionFieldSetItemEntity == null){
|
||||
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
|
||||
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
|
||||
propertyDefinitionFieldSetItemEntity.setOrdinal(ordinal);
|
||||
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
|
||||
}
|
||||
|
||||
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, existingTags));
|
||||
}
|
||||
|
||||
private void addSimpleField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, List<Tag> existingTags){
|
||||
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null);
|
||||
if (propertyDefinitionFieldSetEntity == null) {
|
||||
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
|
||||
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
|
||||
propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity);
|
||||
}
|
||||
|
||||
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null;
|
||||
if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){
|
||||
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
|
||||
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
|
||||
propertyDefinitionFieldSetItemEntity.setOrdinal(0);
|
||||
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
|
||||
} else {
|
||||
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
|
||||
}
|
||||
|
||||
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, existingTags));
|
||||
}
|
||||
|
||||
|
||||
private void addSimpleCommentField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, String comment){
|
||||
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null);
|
||||
if (propertyDefinitionFieldSetEntity == null) {
|
||||
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
|
||||
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
|
||||
propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity);
|
||||
}
|
||||
|
||||
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null;
|
||||
if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){
|
||||
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
|
||||
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
|
||||
propertyDefinitionFieldSetItemEntity.setOrdinal(0);
|
||||
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
|
||||
} else {
|
||||
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
|
||||
}
|
||||
|
||||
propertyDefinitionFieldSetItemEntity.setComment(comment);
|
||||
}
|
||||
|
||||
private FieldEntity buildField(eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, List<Tag> existingTags){
|
||||
FieldEntity fieldEntity = new FieldEntity();
|
||||
String textValue = properties.get(currentField.getId()) != null ? properties.get(currentField.getId()).toString() : null;
|
||||
if (textValue == null || textValue.isEmpty()) return fieldEntity;
|
||||
switch (currentField.getData().getFieldType()){
|
||||
case FREE_TEXT, TEXT_AREA, RICH_TEXT_AREA, RADIO_BOX -> fieldEntity.setTextValue(textValue.trim());
|
||||
case CHECK_BOX, BOOLEAN_DECISION -> fieldEntity.setTextValue(textValue.trim().toLowerCase(Locale.ROOT));
|
||||
case DATE_PICKER -> {
|
||||
Instant instant = null;
|
||||
if(!this.conventionService.isNullOrEmpty(textValue)) {
|
||||
try {
|
||||
instant = Instant.parse(textValue);
|
||||
} catch (DateTimeParseException ex) {
|
||||
instant = LocalDate.parse(textValue).atStartOfDay().toInstant(ZoneOffset.UTC);
|
||||
}
|
||||
}
|
||||
fieldEntity.setDateValue(instant);
|
||||
}
|
||||
case SELECT -> {
|
||||
if(!this.conventionService.isNullOrEmpty(textValue)) {
|
||||
String[] valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, textValue);
|
||||
if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, this.cleanAsObjectString(textValue));
|
||||
fieldEntity.setTextListValue(valuesParsed == null ? List.of(textValue) : Arrays.stream(valuesParsed).toList());
|
||||
}
|
||||
}
|
||||
case DATASET_IDENTIFIER, VALIDATION -> {
|
||||
if(!this.conventionService.isNullOrEmpty(textValue)) {
|
||||
ExternalIdentifierEntity externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, textValue);
|
||||
if (externalIdentifierEntity == null) externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, this.cleanAsObjectString(textValue));
|
||||
if (externalIdentifierEntity == null) throw new MyApplicationException("Could not parse dataset External Identifier : " + textValue);
|
||||
fieldEntity.setExternalIdentifier(externalIdentifierEntity);
|
||||
}
|
||||
}
|
||||
case UPLOAD -> {
|
||||
if(!this.conventionService.isNullOrEmpty(textValue)) {
|
||||
Map<String, String> valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, textValue);
|
||||
if (valuesParsed == null) {
|
||||
valuesParsed = (Map<String, String>) properties.get(currentField.getId());
|
||||
}
|
||||
if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, this.cleanAsObjectString(textValue));
|
||||
if (valuesParsed == null) throw new MyApplicationException("Could not parse upload : " + textValue);
|
||||
String id = valuesParsed.getOrDefault("id", null);
|
||||
if (id == null) throw new MyApplicationException("Could not parse upload : " + textValue);
|
||||
fieldEntity.setTextValue(UUID.fromString(id).toString());
|
||||
}
|
||||
}
|
||||
case CURRENCY -> {
|
||||
if(!this.conventionService.isNullOrEmpty(textValue)) {
|
||||
Currency currency = this.jsonHandlingService.fromJsonSafe(Currency.class, textValue);
|
||||
if (currency == null) currency = this.jsonHandlingService.fromJsonSafe(Currency.class, this.cleanAsObjectString(textValue));
|
||||
//if (currency == null) throw new MyApplicationException("Could not parse Currency : " + textValue);
|
||||
//TODO: {"name":"Euro","value":"EUR"} what we want to keep ?
|
||||
|
||||
fieldEntity.setTextValue(currency == null ? textValue : currency.getName());
|
||||
}
|
||||
}
|
||||
case TAGS -> {
|
||||
if(!this.conventionService.isNullOrEmpty(textValue)) {
|
||||
Tag[] tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, textValue);
|
||||
if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, this.cleanAsObjectString(textValue));
|
||||
if (tags == null) {
|
||||
Tag tag = this.jsonHandlingService.fromJsonSafe(Tag.class, textValue);
|
||||
if (tag == null) tag = this.jsonHandlingService.fromJsonSafe(Tag.class, this.cleanAsObjectString(textValue));
|
||||
if (tag != null) tags = List.of(tag).toArray(Tag[]::new);
|
||||
}
|
||||
if (tags == null)
|
||||
logger.error("Could not parse tag : " + textValue);
|
||||
else {
|
||||
fieldEntity.setTextListValue(new ArrayList<>());
|
||||
for (Tag tag : tags){
|
||||
Tag existingTag = existingTags.stream().filter(x-> x.getName().equalsIgnoreCase(tag.getName())).findFirst().orElse(null);
|
||||
if (existingTag == null){
|
||||
try {
|
||||
UUID.fromString(tag.getId());
|
||||
} catch (IllegalArgumentException e){
|
||||
tag.setId(UUID.randomUUID().toString());
|
||||
}
|
||||
existingTag = tag;
|
||||
existingTags.add(tag);
|
||||
}
|
||||
fieldEntity.getTextListValue().add(existingTag.getId());
|
||||
}
|
||||
}
|
||||
//TODO: should create Description tag maybe should also check the persist
|
||||
//TODO: migrate values maybe we should merge
|
||||
}
|
||||
}
|
||||
|
||||
case INTERNAL_ENTRIES_DMPS -> fieldEntity.setTextValue(textValue.trim());
|
||||
case INTERNAL_ENTRIES_DESCRIPTIONS -> fieldEntity.setTextValue(textValue.trim());
|
||||
case REFERENCE_TYPES -> fieldEntity.setTextValue(textValue.trim());
|
||||
case EXTERNAL_DATASETS -> fieldEntity.setTextValue(textValue.trim());
|
||||
}
|
||||
return fieldEntity;
|
||||
}
|
||||
|
||||
private String cleanAsObjectString(String value){
|
||||
value = value.trim().replace("\\", "");
|
||||
while (!this.conventionService.isNullOrEmpty(value) && value.startsWith("\"")) value = value.substring(1);
|
||||
while (!this.conventionService.isNullOrEmpty(value) && value.endsWith("\"")) value = value.substring(0, value.length() - 1);
|
||||
return value;
|
||||
}
|
||||
|
||||
public static class Currency{
|
||||
private String name;
|
||||
private String value;
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public void setValue(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
}
|
||||
public static class Tag{
|
||||
private String id;
|
||||
private String name;
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,9 +90,13 @@ public class DmpMigrationService {
|
|||
Map<UUID, List<ReferenceTypeFieldEntity>> referenceTypeFieldsByDmpBlueprintId = new HashMap<>();
|
||||
for (DmpBlueprintEntity dmpBlueprintEntity : dmpBlueprintEntities) {
|
||||
eu.eudat.commons.types.dmpblueprint.DefinitionEntity definition = this.xmlHandlingService.fromXml(eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class, dmpBlueprintEntity.getDefinition());
|
||||
List<ReferenceTypeFieldEntity> referenceTypeFieldEntities = definition.getSections().stream().filter(x-> x.getFields() != null).map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.ReferenceType)).map(x-> (ReferenceTypeFieldEntity)x).collect(Collectors.toList());
|
||||
List<ReferenceTypeFieldEntity> referenceTypeFieldEntities = definition.getSections().stream().filter(x-> x.getFields() != null).map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.ReferenceType)).map(x-> (ReferenceTypeFieldEntity)x)
|
||||
.filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.License) || x.getReferenceTypeId().equals(ReferenceTypeIds.Project) || x.getReferenceTypeId().equals(ReferenceTypeIds.Researcher)
|
||||
|| x.getReferenceTypeId().equals(ReferenceTypeIds.Grants) || x.getReferenceTypeId().equals(ReferenceTypeIds.Organizations) || x.getReferenceTypeId().equals(ReferenceTypeIds.Funder))
|
||||
.collect(Collectors.toList());
|
||||
referenceTypeFieldsByDmpBlueprintId.put(dmpBlueprintEntity.getId() , referenceTypeFieldEntities);
|
||||
}
|
||||
|
||||
for (DMP item : items) {
|
||||
//entityManager.detach(item);
|
||||
|
||||
|
|
@ -212,7 +216,6 @@ public class DmpMigrationService {
|
|||
data.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(data);
|
||||
}
|
||||
this.entityManager.flush();
|
||||
}
|
||||
|
||||
private void migrateProjects(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
|
||||
|
|
@ -237,7 +240,6 @@ public class DmpMigrationService {
|
|||
data.setIsActive(IsActive.Active);
|
||||
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
|
||||
this.entityManager.persist(data);
|
||||
this.entityManager.flush();
|
||||
}
|
||||
|
||||
private void migrateResearchers(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
|
||||
|
|
@ -264,7 +266,6 @@ public class DmpMigrationService {
|
|||
data.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(data);
|
||||
}
|
||||
this.entityManager.flush();
|
||||
}
|
||||
|
||||
private void migrateGrantAndFunder(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
|
||||
|
|
@ -306,7 +307,6 @@ public class DmpMigrationService {
|
|||
founder.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(founder);
|
||||
}
|
||||
this.entityManager.flush();
|
||||
}
|
||||
|
||||
private Map<String, UUID> migrateLicense(DMP item, DataManagementPlan model, Map<String, UUID> licenseIdByName, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities) throws JsonProcessingException {
|
||||
|
|
@ -404,7 +404,6 @@ public class DmpMigrationService {
|
|||
entityManager.persist(dmpReferenceEntity);
|
||||
}
|
||||
}
|
||||
this.entityManager.flush();
|
||||
|
||||
return licenseIdByName;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,105 @@
|
|||
package eu.old.eudat.migration;
|
||||
|
||||
import eu.eudat.commons.XmlHandlingService;
|
||||
import eu.eudat.commons.enums.IsActive;
|
||||
import eu.eudat.commons.enums.ReferenceFieldDataType;
|
||||
import eu.eudat.commons.enums.ReferenceSourceType;
|
||||
import eu.eudat.commons.enums.StorageType;
|
||||
import eu.eudat.commons.types.reference.DefinitionEntity;
|
||||
import eu.eudat.commons.types.reference.FieldEntity;
|
||||
import eu.eudat.convention.ConventionService;
|
||||
import eu.eudat.data.ReferenceEntity;
|
||||
import eu.eudat.data.StorageFileEntity;
|
||||
import eu.old.eudat.data.dao.entities.FileUploadDao;
|
||||
import eu.old.eudat.data.dao.entities.ProjectDao;
|
||||
import eu.old.eudat.data.entities.FileUpload;
|
||||
import eu.old.eudat.data.entities.Project;
|
||||
import eu.old.eudat.logic.services.operations.DatabaseRepository;
|
||||
import gr.cite.tools.exception.MyApplicationException;
|
||||
import gr.cite.tools.logging.LoggerService;
|
||||
import jakarta.persistence.EntityManager;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URLConnection;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.UUID;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.util.FileCopyUtils;
|
||||
|
||||
@Service
|
||||
public class StorageFileMigrationService {
|
||||
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(StorageFileMigrationService.class));
|
||||
|
||||
private static final int PageSize = 500;
|
||||
private final DatabaseRepository databaseRepository;
|
||||
private final EntityManager entityManager;
|
||||
private final Environment environment;
|
||||
public StorageFileMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager, Environment environment) {
|
||||
this.databaseRepository = databaseRepository;
|
||||
this.entityManager = entityManager;
|
||||
this.environment = environment;
|
||||
}
|
||||
|
||||
public void migrate() throws IOException {
|
||||
FileUploadDao fileUploadDao = databaseRepository.getFileUploadDao();
|
||||
long total = fileUploadDao.asQueryable().count();
|
||||
logger.debug("Migrate Files Total : " + total);
|
||||
int page = 0;
|
||||
|
||||
List<File> filesToDelete = new ArrayList<>();
|
||||
|
||||
List<FileUpload> items;
|
||||
do {
|
||||
items = fileUploadDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("createdAt"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
|
||||
if (items != null && !items.isEmpty()) {
|
||||
logger.debug("Migrate FileUpload " + page * PageSize + " of " + total);
|
||||
for (FileUpload item : items) {
|
||||
entityManager.detach(item);
|
||||
File file = new File(this.environment.getProperty("file.storage") + item.getId().toString());
|
||||
if (!file.exists()) file = new File(this.environment.getProperty("temp.temp") + item.getId().toString());
|
||||
if (!file.exists()) throw new MyApplicationException("Storage file not exist " + item.getId().toString());
|
||||
|
||||
StorageFileEntity data = new StorageFileEntity();
|
||||
data.setId(item.getId());
|
||||
data.setFileRef(UUID.randomUUID().toString().replaceAll("-", "").toLowerCase(Locale.ROOT));
|
||||
data.setStorageType(StorageType.Main);
|
||||
data.setName(FilenameUtils.removeExtension(item.getName()));
|
||||
data.setExtension(FilenameUtils.getExtension(item.getName()));
|
||||
data.setMimeType(item.getFileType());
|
||||
//data.setMimeType(URLConnection.guessContentTypeFromName(item.getName()));
|
||||
data.setCreatedAt(item.getCreatedAt() == null ? Instant.now() : item.getCreatedAt().toInstant());
|
||||
data.setPurgeAt(null);
|
||||
data.setPurgedAt(null);
|
||||
if (item.getCreator() == null) data.setOwnerId(item.getCreator().getId());
|
||||
|
||||
File destinationFile = new File(Paths.get(this.environment.getProperty("file.mainstorage"), data.getFileRef()).toString());
|
||||
boolean fileCopied = FileCopyUtils.copy(file, destinationFile) > 0;
|
||||
if (!fileCopied) throw new MyApplicationException("Storage file not copied " + data.getId().toString());
|
||||
|
||||
filesToDelete.add(file);
|
||||
|
||||
this.entityManager.persist(data);
|
||||
}
|
||||
this.entityManager.flush();
|
||||
|
||||
page++;
|
||||
}
|
||||
} while (items != null && !items.isEmpty());
|
||||
throw new MyApplicationException("");
|
||||
// for (File file : filesToDelete){
|
||||
// try{
|
||||
// file.delete();
|
||||
// } catch (Exception e){
|
||||
// logger.error(e.getMessage());
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
|
@ -59,7 +59,6 @@ public class TagMigrationService {
|
|||
logger.debug("Migrate Dataset tags " + page * PageSize + " of " + total);
|
||||
|
||||
criteria.setIds(items.stream().map(Dataset::getId).toList());
|
||||
// List<eu.old.eudat.elastic.entities.Dataset> elasticDatasets = this.datasetRepository.queryIds(criteria);
|
||||
List<eu.old.eudat.elastic.entities.Dataset> elasticDatasets = this.datasetRepository.findByIds(items.stream().map(x -> x.getId().toString()).toList());
|
||||
|
||||
for (Dataset item : items) {
|
||||
|
|
@ -72,7 +71,7 @@ public class TagMigrationService {
|
|||
boolean tagAlreadyExists;
|
||||
if (elasticDataset.getTags() != null && !elasticDataset.getTags().isEmpty()) {
|
||||
for(Tag tag : elasticDataset.getTags()) {
|
||||
tagAlreadyExists = savedTagNames.contains(tag.getName());
|
||||
tagAlreadyExists = savedTagNames.contains(tag.getName()); //TODO maybe should check owner to recreate
|
||||
savedTagNames.add(tag.getName());
|
||||
if (!tagAlreadyExists) {
|
||||
TagEntity tagEntity = new TagEntity();
|
||||
|
|
@ -80,6 +79,7 @@ public class TagMigrationService {
|
|||
tagEntity.setLabel(tag.getName());
|
||||
tagEntity.setCreatedAt(Instant.now());
|
||||
tagEntity.setUpdatedAt(Instant.now());
|
||||
if (item.getCreator() != null) tagEntity.setCreatedById(item.getCreator().getId());
|
||||
tagEntity.setIsActive(IsActive.Active);
|
||||
savedTagIdsByName.put(tag.getName(), tagEntity.getId());
|
||||
this.entityManager.persist(tagEntity);
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ public class MigrationController {
|
|||
private final DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService;
|
||||
private final DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService;
|
||||
private final DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService;
|
||||
private final StorageFileMigrationService storageFileMigrationService;
|
||||
|
||||
public MigrationController(
|
||||
DmpMigrationService dmpMigrationService,
|
||||
|
|
@ -62,7 +63,7 @@ public class MigrationController {
|
|||
DmpUserMigrationService dmpUserMigrationService,
|
||||
UserMigrationService userMigrationService,
|
||||
DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService,
|
||||
DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService, DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService) {
|
||||
DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService, DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService, StorageFileMigrationService storageFileMigrationService) {
|
||||
this.dmpMigrationService = dmpMigrationService;
|
||||
this.datasetMigrationService = datasetMigrationService;
|
||||
this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService;
|
||||
|
|
@ -83,6 +84,7 @@ public class MigrationController {
|
|||
this.dmpBlueprintXmlMigrationService = dmpBlueprintXmlMigrationService;
|
||||
this.descriptionTemplateXmlMigrationService = descriptionTemplateXmlMigrationService;
|
||||
this.descriptionTemplateXmlCleanInvalidReferenceTypesService = descriptionTemplateXmlCleanInvalidReferenceTypesService;
|
||||
this.storageFileMigrationService = storageFileMigrationService;
|
||||
}
|
||||
|
||||
@GetMapping("all")
|
||||
|
|
@ -112,7 +114,9 @@ public class MigrationController {
|
|||
|
||||
this.dmpUserMigrationService.migrate();
|
||||
|
||||
this.referenceMigrationService.migrateDatasetReferences(); //TODO
|
||||
//this.referenceMigrationService.migrateDatasetReferences(); //TODO
|
||||
|
||||
this.storageFileMigrationService.migrate();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
@ -125,6 +129,14 @@ public class MigrationController {
|
|||
|
||||
}
|
||||
|
||||
@GetMapping("storage-files")
|
||||
@Transactional
|
||||
public boolean migrateStorageFile() throws IOException {
|
||||
this.storageFileMigrationService.migrate();
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
@GetMapping("description-template-xml")
|
||||
@Transactional
|
||||
public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
|
||||
|
|
|
|||
Loading…
Reference in New Issue