migration changes

dmp-backend/core/pom.xml

@@ -57,6 +57,34 @@
             <artifactId>elastic</artifactId>
             <version>2.1.2</version>
         </dependency>
+        <dependency>
+            <groupId>gr.cite</groupId>
+            <artifactId>queue-inbox</artifactId>
+            <version>1.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>gr.cite</groupId>
+            <artifactId>queue-outbox</artifactId>
+            <version>1.0.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>gr.cite</groupId>
+            <artifactId>oidc-authn</artifactId>
+            <version>2.2.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>gr.cite</groupId>
+            <artifactId>keycloak-api</artifactId>
+            <version>1.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>gr.cite</groupId>
+            <artifactId>data-tools</artifactId>
+            <version>2.1.2</version>
+        </dependency>
+
         <dependency>
 			<groupId>org.eclipse.angus</groupId>
 			<artifactId>jakarta.mail</artifactId>

dmp-backend/core/src/main/java/eu/eudat/data/BaseEntity.java

@@ -1,5 +0,0 @@
-package eu.eudat.data;
-
-public interface BaseEntity {
-
-}

dmp-backend/core/src/main/java/eu/eudat/service/keycloak/KeycloakService.java

@@ -1,8 +1,6 @@
 package eu.eudat.service.keycloak;
 
-import eu.eudat.data.TenantEntity;
 import org.jetbrains.annotations.NotNull;
-import org.keycloak.representations.idm.GroupRepresentation;
 
 import java.util.HashMap;
 import java.util.UUID;

dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java

@@ -4,16 +4,20 @@ import eu.eudat.commons.JsonHandlingService;
 import eu.eudat.commons.XmlHandlingService;
 import eu.eudat.commons.enums.DescriptionStatus;
 import eu.eudat.commons.enums.IsActive;
+import eu.eudat.commons.enums.ReferenceFieldDataType;
+import eu.eudat.commons.enums.ReferenceSourceType;
 import eu.eudat.commons.types.description.*;
 import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity;
+import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity;
 import eu.eudat.commons.types.dmpblueprint.DefinitionEntity;
+import eu.eudat.commons.types.externalfetcher.ExternalFetcherBaseSourceConfigurationEntity;
+import eu.eudat.commons.types.referencetype.ReferenceTypeDefinitionEntity;
 import eu.eudat.convention.ConventionService;
 import eu.eudat.data.*;
 import eu.eudat.model.Dmp;
-import eu.eudat.query.DescriptionTemplateQuery;
+import eu.eudat.model.Reference;
-import eu.eudat.query.DmpBlueprintQuery;
+import eu.eudat.model.ReferenceType;
-import eu.eudat.query.DmpDescriptionTemplateQuery;
+import eu.eudat.query.*;
-import eu.eudat.query.DmpQuery;
 import eu.old.eudat.data.dao.entities.DatasetDao;
 import eu.old.eudat.data.entities.Dataset;
 import eu.old.eudat.logic.services.operations.DatabaseRepository;
@@ -65,6 +69,18 @@ public class DatasetMigrationService {
         logger.debug("Migrate Dataset Total : " + total);
         int page = 0;
 
+        Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap = new HashMap<>();
+        List<ReferenceTypeEntity> referenceTypes = this.queryFactory.query(ReferenceTypeQuery.class).collectAs(new BaseFieldSet().ensure(ReferenceType._id).ensure(ReferenceType._definition));
+        for (ReferenceTypeEntity referenceType : referenceTypes){
+            referenceTypeDefinitionEntityMap.put(referenceType.getId(), this.xmlHandlingService.fromXml(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition()));
+        }
+
+        Map<ReferenceKey, ReferenceEntity> referenceMap = new HashMap<>();
+        List<ReferenceEntity> references = this.queryFactory.query(ReferenceQuery.class).collectAs(new BaseFieldSet().ensure(Reference._id).ensure(Reference._source).ensure(Reference._type).ensure(Reference._reference));
+        for (ReferenceEntity reference : references){
+            if (reference.getCreatedById() == null) referenceMap.put(new ReferenceKey(reference), reference);
+        }
+        
         List<Dataset> items;
         do {
             items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
@@ -131,17 +147,13 @@ public class DatasetMigrationService {
                     }
 
                     eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity = descriptionTemplateDefinitionMap.getOrDefault(item.getProfile().getId(), null);
-                    List<Tag> existingTags = new ArrayList<>();
+                    data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap)));
-                    
-                    data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, existingTags)));
 
                     if (data.getCreatedById() == null){
                         logger.error("Migration skipped creator not found " + item.getId());
                         throw  new MyApplicationException("Migration skipped creator not found " + item.getId());
                     }
                     
-                    this.persistTags(existingTags, data);
-
                     this.entityManager.persist(data);
                     this.entityManager.flush();
                 }
@@ -152,19 +164,6 @@ public class DatasetMigrationService {
         throw new MyApplicationException("");
     }
     
-    private void persistTags(List<Tag> existingTags, DescriptionEntity data){
-        for (Tag tag : existingTags) {
-            TagEntity tagEntity = new TagEntity();
-            tagEntity.setId(UUID.fromString(tag.getId()));
-            tagEntity.setLabel(tag.getName());
-            tagEntity.setCreatedAt(Instant.now());
-            tagEntity.setUpdatedAt(Instant.now());
-            tagEntity.setCreatedById(data.getCreatedById());
-            tagEntity.setIsActive(IsActive.Active);
-            this.entityManager.persist(tagEntity);
-        }
-    }
-    
     private List<DmpDescriptionTemplateEntity> getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities){
         List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
         if (itemDescriptionTemplates.isEmpty()) {
@@ -187,7 +186,7 @@ public class DatasetMigrationService {
         return itemDescriptionTemplates;
     }
     
-    private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, List<Tag> existingTags) {
+    private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap) {
         if (this.conventionService.isNullOrEmpty(item.getProperties())) return null;
         JSONObject jObject = new JSONObject(item.getProperties());
         Map<String, Object> properties = jObject.toMap();
@@ -225,7 +224,7 @@ public class DatasetMigrationService {
                         continue;
                     }
 
-                    this.addSimpleField(propertyDefinitionEntity, currentFieldSet, currentField, properties, existingTags);
+                    this.addSimpleField(propertyDefinitionEntity, currentFieldSet, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap);
                 }
             }
         }
@@ -292,13 +291,13 @@ public class DatasetMigrationService {
                 }
                 if (propertyDefinitionFieldSetEntity == null)  throw new MyApplicationException("Invalid multiple key group " + key);
 
-                this.addMultipleField(propertyDefinitionFieldSetEntity, ordinal, currentField, properties, existingTags);
+                this.addMultipleField(propertyDefinitionFieldSetEntity, ordinal, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap);
             }
         }
         return propertyDefinitionEntity;
     }
     
-    private void addMultipleField(PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, List<Tag> existingTags){
+    private void addMultipleField(PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
         PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null);
 
         if (propertyDefinitionFieldSetItemEntity == null){
@@ -308,10 +307,10 @@ public class DatasetMigrationService {
             propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
         }
 
-        propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, existingTags));
+        propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
     }
     
-    private void addSimpleField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, List<Tag> existingTags){
+    private void addSimpleField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
         PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null);
         if (propertyDefinitionFieldSetEntity == null) {
             propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
@@ -329,7 +328,7 @@ public class DatasetMigrationService {
             propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
         }
 
-        propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, existingTags));
+        propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
     }
 
 
@@ -354,7 +353,7 @@ public class DatasetMigrationService {
         propertyDefinitionFieldSetItemEntity.setComment(comment);
     }
     
-    private FieldEntity buildField(eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, List<Tag> existingTags){
+    private FieldEntity buildField(eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
         FieldEntity fieldEntity = new FieldEntity();
         String textValue = properties.get(currentField.getId()) != null ? properties.get(currentField.getId()).toString() : null;
         if (textValue == null || textValue.isEmpty()) return fieldEntity;
@@ -404,10 +403,20 @@ public class DatasetMigrationService {
                 if(!this.conventionService.isNullOrEmpty(textValue)) {
                     Currency currency = this.jsonHandlingService.fromJsonSafe(Currency.class, textValue);
                     if (currency == null) currency = this.jsonHandlingService.fromJsonSafe(Currency.class, this.cleanAsObjectString(textValue));
-                    //if (currency == null) throw new MyApplicationException("Could not parse Currency : " + textValue);
+                    if (currency == null && textValue.toUpperCase(Locale.ROOT).contains("EUR")) {
+                        currency = new Currency();
+                        currency.setName("Euro");
+                        currency.setValue("EUR");
+                    }
+                    if (currency == null && textValue.toUpperCase(Locale.ROOT).contains("US DOLLAR")) {
+                        currency = new Currency();
+                        currency.setName("US DOLLAR");
+                        currency.setValue("USD");
+                    }
+                    if (currency == null) throw new MyApplicationException("Could not parse Currency : " + textValue);
                     //TODO: {"name":"Euro","value":"EUR"} what we want to keep ?
                     
-                    fieldEntity.setTextValue(currency  == null ? textValue : currency.getName());
+                    fieldEntity.setTextValue(currency.getName());
                 }
             }
             case TAGS -> {
@@ -419,39 +428,120 @@ public class DatasetMigrationService {
                         if (tag == null) tag = this.jsonHandlingService.fromJsonSafe(Tag.class, this.cleanAsObjectString(textValue));
                         if (tag != null) tags = List.of(tag).toArray(Tag[]::new);
                     }
-                    if (tags == null)
+                    if (tags == null) {
+                        //throw new MyApplicationException("Could not parse tag : " + textValue); //TODO
                         logger.error("Could not parse tag : " + textValue);
-                    else {
+                        break;
-                        fieldEntity.setTextListValue(new ArrayList<>());
-                        for (Tag tag : tags){
-                            Tag existingTag = existingTags.stream().filter(x-> x.getName().equalsIgnoreCase(tag.getName())).findFirst().orElse(null);
-                            if (existingTag == null){
-                                try {
-                                    UUID.fromString(tag.getId());
-                                } catch (IllegalArgumentException e){
-                                    tag.setId(UUID.randomUUID().toString());
                     }
-                                existingTag = tag;
+                    fieldEntity.setTextListValue(Arrays.stream(tags).toList().stream().map(Tag::getName).toList());
-                                existingTags.add(tag);
-                            }
-                            fieldEntity.getTextListValue().add(existingTag.getId());
-                        }
-                    }
-                    //TODO: should create Description tag maybe should also check the persist 
-                    //TODO: migrate values maybe we should merge
                 }
             }
             
-            case INTERNAL_ENTRIES_DMPS -> fieldEntity.setTextValue(textValue.trim());
+            case INTERNAL_ENTRIES_DMPS -> throw new MyApplicationException("Found INTERNAL_ENTRIES_DMPS into description");
-            case INTERNAL_ENTRIES_DESCRIPTIONS -> fieldEntity.setTextValue(textValue.trim());
+            case INTERNAL_ENTRIES_DESCRIPTIONS -> throw new MyApplicationException("Found INTERNAL_ENTRIES_DMPS into description");
-            case REFERENCE_TYPES -> fieldEntity.setTextValue(textValue.trim());
+            case REFERENCE_TYPES -> {
-            case EXTERNAL_DATASETS -> fieldEntity.setTextValue(textValue.trim());
+                if(!this.conventionService.isNullOrEmpty(textValue)) {
+                    Map<String, Object>[] references = this.jsonHandlingService.fromJsonSafe(Map[].class, textValue);
+                    if (references == null) references = this.jsonHandlingService.fromJsonSafe(Map[].class, this.cleanAsObjectString(textValue));
+                    if (references == null) {
+                        Map<String, Object> reference = this.jsonHandlingService.fromJsonSafe(Map.class, textValue);
+                        if (reference == null) reference = this.jsonHandlingService.fromJsonSafe(Map.class, this.cleanAsObjectString(textValue));
+                        if (reference != null) references = List.of(reference).toArray(Map[]::new);
+                    }
+                    if (references == null){
+                        logger.warn("Could not parse reference : " + textValue);
+                        //throw new MyApplicationException("Could not parse reference : " + textValue);
+                        break;
+                    }
+                    ReferenceTypeDataEntity referenceTypeDataEntity = (ReferenceTypeDataEntity)currentField.getData();
+                    if (referenceTypeDataEntity == null) throw new MyApplicationException("Could not parse description template reference : " + textValue);
+                    
+                    fieldEntity.setTextListValue(new ArrayList<>());
+                    for (Map<String, Object> item : references){
+                        ReferenceEntity referenceEntity = this.geReferenceEntity(item, referenceTypeDefinitionEntityMap, referenceTypeDataEntity, textValue, referenceMap);
+                        fieldEntity.getTextListValue().add(referenceEntity.getId().toString());
+                    }
+                }
+            }
         }
         return fieldEntity;
     }
     
+    private ReferenceEntity geReferenceEntity(Map<String, Object> item, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, ReferenceTypeDataEntity referenceTypeDataEntity, String textValue, Map<ReferenceKey, ReferenceEntity> referenceMap){
+        ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null);
+        if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + textValue);
+        if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + textValue);
+
+        ReferenceEntity data = new ReferenceEntity();
+        if (item.containsKey("name"))  data.setLabel((String) item.get("name"));
+        if (this.conventionService.isNullOrEmpty(data.getLabel()) && item.containsKey("label"))  data.setLabel((String) item.get("label"));
+        if (item.containsKey("abbreviation")) data.setAbbreviation((String) item.get("abbreviation"));
+        if (item.containsKey("id")) data.setReference((String)item.get("id"));
+        if (item.containsKey("reference")) data.setReference((String)item.get("reference"));
+        if (this.conventionService.isNullOrEmpty(data.getReference()) && item.containsKey("pid"))  data.setReference((String)item.get("pid"));
+        
+        data.setId(UUID.randomUUID());
+        data.setCreatedAt(Instant.now());
+        data.setUpdatedAt(Instant.now());
+        data.setSourceType(ReferenceSourceType.External);
+        data.setTypeId(referenceTypeDataEntity.getReferenceTypeId());
+        data.setIsActive(IsActive.Active);
+        eu.eudat.commons.types.reference.DefinitionEntity definitionEntity = new eu.eudat.commons.types.reference.DefinitionEntity();
+        definitionEntity.setFields(new ArrayList<>());
+        for (Map.Entry<String, Object> entries : item.entrySet()){
+            if (entries.getValue() != null && !entries.getKey().equals("name") && !entries.getKey().equals("label")&& !entries.getKey().equals("abbreviation")
+                    && !entries.getKey().equals("id") && !entries.getKey().equals("pid")&& !entries.getKey().equals("source")) {
+                eu.eudat.commons.types.reference.FieldEntity field = new eu.eudat.commons.types.reference.FieldEntity();
+                field.setCode(entries.getKey());
+                field.setDataType(ReferenceFieldDataType.Text);
+                field.setValue(entries.getValue().toString());
+                definitionEntity.getFields().add(field);
+            }
+        }
+        if (!this.conventionService.isListNullOrEmpty(definitionEntity.getFields())) data.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
+
+
+        if (this.conventionService.isNullOrEmpty(data.getLabel())) data.setLabel(data.getReference());
+        if (referenceTypeDefinition.getSources().size() == 1) data.setSource(referenceTypeDefinition.getSources().getFirst().getKey());
+        else {
+            String key = referenceTypeDefinition.getSources().stream().filter(x-> x.getKey().equalsIgnoreCase((String) item.getOrDefault("source", null))).map(ExternalFetcherBaseSourceConfigurationEntity::getKey).findFirst().orElse(null);
+            if (this.conventionService.isNullOrEmpty(key)) {
+                logger.warn("Select first source  for reference : " + textValue);
+                data.setSourceType(ReferenceSourceType.Internal);
+                data.setSource(referenceTypeDefinition.getSources().getFirst().getKey());
+                //throw new MyApplicationException("Could not find reference source for reference : " + textValue); //TODO
+            } else {
+                data.setSource(key);
+            }
+        }
+        if (this.conventionService.isNullOrEmpty(data.getReference())) {
+            data.setReference(this.queryFactory.query(ReferenceQuery.class).sources(data.getSource()).like(data.getLabel()).typeIds(data.getTypeId()).collectAs(new BaseFieldSet().ensure(Reference._reference, Reference._label))
+                    .stream().filter(x-> x.getLabel().equals(data.getLabel())).map(ReferenceEntity::getReference).findFirst().orElse(null));//TODO
+        }
+        if (this.conventionService.isNullOrEmpty(data.getReference())) {
+            logger.warn("Local reference created with random id" + textValue);
+            data.setReference(data.getId().toString().replace("-", ""));
+            data.setSourceType(ReferenceSourceType.Internal); //TODO
+        }
+        
+        ReferenceEntity existingEntity = referenceMap.getOrDefault(new ReferenceKey(data), null);
+        if(existingEntity == null){
+            referenceMap.put(new ReferenceKey(data), data);
+            this.entityManager.persist(data);
+            return data;
+        } else {
+            return existingEntity;
+        }
+        
+    }
+    
     private String cleanAsObjectString(String value){
         value = value.trim().replace("\\", "");
+        value = value.trim().replace("[\"{", "[{");
+        value = value.trim().replace("}\"]", "}]");
+        value = value.trim().replace("}\", \"{", "}, {");
+        value = value.trim().replace("}\",\"{", "},{");
+    
         while (!this.conventionService.isNullOrEmpty(value) && value.startsWith("\"")) value = value.substring(1);
         while (!this.conventionService.isNullOrEmpty(value) && value.endsWith("\"")) value = value.substring(0, value.length() - 1);
         return value;
@@ -497,4 +587,46 @@ public class DatasetMigrationService {
             this.name = name;
         }
     }
+
+    public class ReferenceKey {
+        private final UUID type;
+        private final String source;
+        private final String reference;
+        private final int hashCode;
+
+
+        public ReferenceKey(ReferenceEntity reference) {
+            this.type = reference.getTypeId();
+            this.source = reference.getSource().trim();
+            this.reference = reference.getReference().trim();
+            hashCode = Objects.hash(this.type, this.source, this.reference);
+        }
+
+        public UUID getType() {
+            return type;
+        }
+
+        public String getSource() {
+            return source;
+        }
+
+        public String getReference() {
+            return reference;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+            ReferenceKey that = (ReferenceKey) o;
+            return Objects.equals(type, that.getType()) && Objects.equals(source, that.getSource()) && Objects.equals(reference, that.getReference());
+        }
+
+        @Override
+        public int hashCode() {
+            return this.hashCode;
+        }
+    }
 }

dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java

@@ -4,11 +4,13 @@ import eu.eudat.commons.XmlHandlingService;
 import eu.eudat.commons.enums.*;
 import eu.eudat.commons.types.descriptiontemplate.*;
 import eu.eudat.commons.types.descriptiontemplate.fielddata.*;
+import eu.eudat.commons.types.externalfetcher.*;
 import eu.eudat.commons.types.referencetype.*;
 import eu.eudat.convention.ConventionService;
 import eu.eudat.data.DescriptionTemplateEntity;
 import eu.eudat.data.ReferenceEntity;
 import eu.eudat.data.ReferenceTypeEntity;
+import eu.eudat.file.transformer.enums.FieldDataExternalDatasetType;
 import eu.eudat.model.DescriptionTemplate;
 import eu.eudat.query.DescriptionTemplateQuery;
 import eu.old.eudat.logic.utilities.builders.XmlBuilder;
@@ -199,8 +201,23 @@ public class DescriptionTemplateXmlMigrationService {
             
         }).toList());
         data.setIncludeInExport(persist.getExport());
-        if (persist.getData() != null)
+        if (persist.getData() != null) {
             data.setData(this.buildFieldDataEntity(persist.getData(), persist.getViewStyle().getRenderStyle(), referenceTypeEntityMap));
+            if (persist.getViewStyle().getRenderStyle().equals("externalDatasets")){
+                FieldDataExternalDatasetType type = null;
+                try {
+                    type = FieldDataExternalDatasetType.of(((ExternalDatasetsData)persist.getData()).getType());
+                } catch (Exception e) { type = FieldDataExternalDatasetType.Other; }
+                if (type == null) type = FieldDataExternalDatasetType.Other;
+                if (data.getSchematics() == null) data.setSchematics(new ArrayList<>());
+                switch (type){
+                    case Other -> data.getSchematics().add("referencetype.externaldataset.type.other");
+                    case ProducedDataset -> data.getSchematics().add("referencetype.externaldataset.type.produced");
+                    case ReusedDataset -> data.getSchematics().add("referencetype.externaldataset.type.reused");
+                    default -> throw new MyApplicationException("Invalid type " + type);
+                }
+            }
+        }
         
         if (persist.getVisible() != null && !this.conventionService.isListNullOrEmpty(persist.getVisible().getRules())) {
             data.setVisibilityRules(new ArrayList<>());
@@ -284,12 +301,7 @@ public class DescriptionTemplateXmlMigrationService {
         }
 
         if (renderStyle.equals("externalDatasets")){
-            ExternalDatasetDataEntity data = new ExternalDatasetDataEntity();
+            return this.buildReferenceTypeDataEntity(persist.getLabel(), ((ExternalDatasetsData)persist).getMultiAutoComplete(), ReferenceTypeIds.Datasets);
-            data.setLabel(persist.getLabel());
-            data.setMultipleSelect(((ExternalDatasetsData)persist).getMultiAutoComplete());
-            data.setType(FieldDataExternalDatasetType.of(((ExternalDatasetsData)persist).getType()));
-            data.setFieldType(FieldType.EXTERNAL_DATASETS);
-            return data;
         }
 
 
@@ -454,7 +466,7 @@ public class DescriptionTemplateXmlMigrationService {
     public String referenceTypeCalculatedHash(ReferenceTypeDefinitionEntity definitionEntity) {
         List<String> sourceKeys = new ArrayList<>();
         if (definitionEntity.getSources() != null){
-            for (ReferenceTypeSourceExternalApiConfigurationEntity source : definitionEntity.getSources().stream().map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList()){
+            for (ExternalFetcherApiSourceConfigurationEntity source : definitionEntity.getSources().stream().map(x-> (ExternalFetcherApiSourceConfigurationEntity)x).toList()){
                 StringBuilder sourceStringBuilder = new StringBuilder();
                 sourceStringBuilder.append(source.getUrl());
                 sourceStringBuilder.append(source.getHttpMethod().getValue().toString());
@@ -489,10 +501,10 @@ public class DescriptionTemplateXmlMigrationService {
         return data;
     }
 
-    private @NotNull ReferenceTypeSourceBaseConfigurationEntity buildSourceBaseConfigEntity(AutoCompleteData.AutoCompleteSingleData persist, int ordinal) throws URISyntaxException {
+    private @NotNull ExternalFetcherApiSourceConfigurationEntity buildSourceBaseConfigEntity(AutoCompleteData.AutoCompleteSingleData persist, int ordinal) throws URISyntaxException {
-        if (persist == null) return new ReferenceTypeSourceExternalApiConfigurationEntity();
+        if (persist == null) return new ExternalFetcherApiSourceConfigurationEntity();
 
-        ReferenceTypeSourceExternalApiConfigurationEntity apiEntity = new ReferenceTypeSourceExternalApiConfigurationEntity();
+        ExternalFetcherApiSourceConfigurationEntity apiEntity = new ExternalFetcherApiSourceConfigurationEntity();
 
         URI uri;
         if (persist.getUrl().contains("?")) {

dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java

@@ -90,33 +90,33 @@ public class MigrationController {
     @GetMapping("all")
     @Transactional
     public boolean migrateAll() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException, NoSuchFieldException, InvalidApplicationException, TransformerException, URISyntaxException {
-        this.dataRepositoryMigrationService.migrate();
+//        this.dataRepositoryMigrationService.migrate();
-        this.externalDatasetMigrationService.migrate();
+//        this.externalDatasetMigrationService.migrate();
-        this.funderMigrationService.migrate();
+//        this.funderMigrationService.migrate();
-        this.grantMigrationService.migrate();
+//        this.grantMigrationService.migrate();
-        this.organizationMigrationService.migrate();
+//        this.organizationMigrationService.migrate();
-        this.projectMigrationService.migrate();
+//        this.projectMigrationService.migrate();
-        this.registryMigrationService.migrate();
+//        this.registryMigrationService.migrate();
-        this.researcherMigrationService.migrate();
+//        this.researcherMigrationService.migrate();
-        this.serviceMigrationService.migrate();
+//        this.serviceMigrationService.migrate();
-
+//
-        this.userContactInfoMigrationService.migrate();
+//        this.userContactInfoMigrationService.migrate();
-        this.userMigrationService.migrate();
+//        this.userMigrationService.migrate();
-
+//
-        //XML recreate
+//        //XML recreate
-        this.dmpBlueprintXmlMigrationService.migrate();
+//        this.dmpBlueprintXmlMigrationService.migrate();
-        this.descriptionTemplateXmlMigrationService.migrate();
+//        this.descriptionTemplateXmlMigrationService.migrate();
-
+//
-        this.dmpMigrationService.migrate();
+//        this.dmpMigrationService.migrate();
-        this.dmpDatasetProfileMigrationService.migrate();
+//        this.dmpDatasetProfileMigrationService.migrate();
-        //this.datasetMigrationService.migrate(); //TODO
+        this.datasetMigrationService.migrate(); //TODO
-        this.tagMigrationService.migrate();
+//        this.tagMigrationService.migrate();
-        
+//        
-        this.dmpUserMigrationService.migrate();
+//        this.dmpUserMigrationService.migrate();
-        
+//        
-        //this.referenceMigrationService.migrateDatasetReferences(); //TODO
+//        this.referenceMigrationService.migrateDatasetReferences(); 
-
+//
-        this.storageFileMigrationService.migrate();
+//        this.storageFileMigrationService.migrate();
         
         return true;
     }