diff --git a/dmp-db-scema/updates/00.01.010_Align_DescriptionTemplate_table.sql b/dmp-db-scema/updates/00.01.010_Align_DescriptionTemplate_table.sql index 8370813ec..7ab4c8784 100644 --- a/dmp-db-scema/updates/00.01.010_Align_DescriptionTemplate_table.sql +++ b/dmp-db-scema/updates/00.01.010_Align_DescriptionTemplate_table.sql @@ -54,7 +54,7 @@ BEGIN ) ); - UPDATE public."DescriptionTemplate" SET version_status = 2 where status = 0 id in ( + UPDATE public."DescriptionTemplate" SET version_status = 2 where status = 0 and id in ( select dt.id from public."DescriptionTemplate" as dt where dt.version = ( diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java index 2b96de823..675eaa227 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java @@ -248,7 +248,6 @@ public class DatasetMigrationService { for (String key : properties.keySet()) { if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) { String newKey = key.trim().substring("multiple_".length()); - //newKey = newKey.replace("multiple_formats-to-plan-to-store_fa87dc73-d0b6-b707-c632-369f4191a306_", "");//TODO String[] keyParts = newKey.split("_"); if (keyParts.length > 4 && newKey.contains("_multiple_")){ String[] multiParts = newKey.split("_multiple_"); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java index b12fae81e..c8bac8079 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java @@ -1,5 +1,6 @@ package eu.old.eudat.migration; +import com.fasterxml.jackson.annotation.JsonValue; import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.enums.*; import eu.eudat.commons.types.descriptiontemplate.*; @@ -10,7 +11,7 @@ import eu.eudat.convention.ConventionService; import eu.eudat.data.DescriptionTemplateEntity; import eu.eudat.data.ReferenceEntity; import eu.eudat.data.ReferenceTypeEntity; -import eu.eudat.file.transformer.enums.FieldDataExternalDatasetType; +import eu.eudat.data.converters.enums.DatabaseEnum; import eu.eudat.model.DescriptionTemplate; import eu.eudat.query.DescriptionTemplateQuery; import eu.old.eudat.logic.utilities.builders.XmlBuilder; @@ -618,4 +619,26 @@ public class DescriptionTemplateXmlMigrationService { throw new MyApplicationException("Migrate DescriptionTemplate autocomplete method is invalid " + method); } } + + public enum FieldDataExternalDatasetType implements DatabaseEnum { + ReusedDataset("reused_dataset"), + ProducedDataset("produced_dataset"), + Other("other"); + private final String value; + + FieldDataExternalDatasetType(String value) { + this.value = value; + } + + @JsonValue + public String getValue() { + return value; + } + + private static final Map map = EnumUtils.getEnumValueMap(FieldDataExternalDatasetType.class); + + public static FieldDataExternalDatasetType of(String i) { + return map.get(i); + } + } } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java index 5d2bb1828..a94cef23c 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java @@ -1,13 +1,11 @@ package eu.old.eudat.migration; -import eu.eudat.commons.JsonHandlingService; -import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.enums.IsActive; import eu.eudat.data.DescriptionTagEntity; +import eu.eudat.data.ReferenceEntity; import eu.eudat.data.TagEntity; import eu.old.eudat.data.dao.entities.DatasetDao; import eu.old.eudat.data.entities.Dataset; -import eu.old.eudat.elastic.criteria.DatasetCriteria; import eu.old.eudat.elastic.entities.Tag; import eu.old.eudat.elastic.repository.DatasetRepository; import eu.old.eudat.logic.services.operations.DatabaseRepository; @@ -29,15 +27,13 @@ public class TagMigrationService { private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(TagMigrationService.class)); private final DatabaseRepository databaseRepository; - private final QueryFactory queryFactory; private final DatasetRepository datasetRepository; private static final int PageSize = 500; private static final boolean TestMode = false; private final EntityManager entityManager; - public TagMigrationService(DatabaseRepository databaseRepository, QueryFactory queryFactory, DatasetRepository datasetRepository, EntityManager entityManager) { + public TagMigrationService(DatabaseRepository databaseRepository, DatasetRepository datasetRepository, EntityManager entityManager) { this.databaseRepository = databaseRepository; - this.queryFactory = queryFactory; this.datasetRepository = datasetRepository; this.entityManager = entityManager; } @@ -48,8 +44,8 @@ public class TagMigrationService { logger.debug("Migrate Tags for Dataset Total : " + total); int page = 0; - Set savedTagNames = new HashSet<>(); - Map savedTagIdsByName = new HashMap<>(); + Map savedTagIdsByName = new HashMap<>(); + HashSet existingTagIds = new HashSet<>(); List items; do { @@ -69,27 +65,28 @@ public class TagMigrationService { boolean tagAlreadyExists; if (elasticDataset.getTags() != null && !elasticDataset.getTags().isEmpty()) { for (Tag tag : elasticDataset.getTags()) { - tagAlreadyExists = savedTagNames.contains(tag.getName()); //TODO maybe should check owner to recreate - savedTagNames.add(tag.getName()); + tagAlreadyExists = savedTagIdsByName.containsKey(new TagKey(item, tag)); //TODO we want owner logic ? if (!tagAlreadyExists) { TagEntity tagEntity = new TagEntity(); - tagEntity.setId(UUID.fromString(tag.getId())); + if (!existingTagIds.contains(UUID.fromString(tag.getId()))) tagEntity.setId(UUID.fromString(tag.getId())); + else tagEntity.setId(UUID.randomUUID()); tagEntity.setLabel(tag.getName()); tagEntity.setCreatedAt(Instant.now()); tagEntity.setUpdatedAt(Instant.now()); if (item.getCreator() != null) tagEntity.setCreatedById(item.getCreator().getId()); tagEntity.setIsActive(IsActive.Active); - savedTagIdsByName.put(tag.getName(), tagEntity.getId()); + savedTagIdsByName.put(new TagKey(item, tag), tagEntity.getId()); this.entityManager.persist(tagEntity); } DescriptionTagEntity descriptionTagEntity = new DescriptionTagEntity(); descriptionTagEntity.setId(UUID.randomUUID()); - descriptionTagEntity.setTagId(savedTagIdsByName.get(tag.getName())); + descriptionTagEntity.setTagId(savedTagIdsByName.get(new TagKey(item, tag))); descriptionTagEntity.setDescriptionId(item.getId()); descriptionTagEntity.setCreatedAt(Instant.now()); descriptionTagEntity.setUpdatedAt(Instant.now()); descriptionTagEntity.setIsActive(IsActive.Active); this.entityManager.persist(descriptionTagEntity); + existingTagIds.add(descriptionTagEntity.getTagId()); } } this.entityManager.flush(); @@ -100,4 +97,41 @@ public class TagMigrationService { } while (items != null && !items.isEmpty() && !TestMode); } + public static class TagKey { + private final UUID owner; + private final String name; + private final int hashCode; + + + public TagKey(Dataset item, Tag tag) { + this.name = tag.getName(); + if (item.getCreator() != null) this.owner = item.getCreator().getId(); + else this.owner = null; + hashCode = Objects.hash(this.owner, this.name); + } + + public UUID getOwner() { + return owner; + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + TagKey that = (TagKey) o; + return Objects.equals(owner, that.getOwner()) && Objects.equals(name, that.getName()); + } + + @Override + public int hashCode() { + return this.hashCode; + } + } + } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java index 729837f3a..c6b3ee0b8 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java @@ -1,5 +1,6 @@ package eu.old.eudat.publicapi.migration; +import com.fasterxml.jackson.core.JsonProcessingException; import eu.old.eudat.migration.*; import io.swagger.annotations.Api; import jakarta.xml.bind.JAXBException; @@ -112,7 +113,7 @@ public class MigrationController { this.dmpMigrationService.migrate(); this.dmpDatasetProfileMigrationService.migrate(); this.dmpUserMigrationService.migrate(); - + //Description this.datasetMigrationService.migrate(); this.datasetReferenceMigrationService.migrateDatasetReferences(); @@ -124,174 +125,174 @@ public class MigrationController { return true; } -// @GetMapping("dmp-blueprints-xml") -// @Transactional -// public boolean migrateDmpBlueprint() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException { -// this.dmpBlueprintXmlMigrationService.migrate(); -// return true; -// -// } -// -// @GetMapping("storage-files") -// @Transactional -// public boolean migrateStorageFile() throws IOException { -// this.storageFileMigrationService.migrate(); -// return true; -// -// } -// -// @GetMapping("description-template-xml") -// @Transactional -// public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { -// this.descriptionTemplateXmlMigrationService.migrate(); -// return true; -// -// } -// -// @GetMapping("description-template-xml-clean-invalid-reference-types") -// @Transactional -// public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { -// this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate(); -// return true; -// -// } -// -// @GetMapping("contacts") -// @Transactional -// public boolean migrateContacts() { -// this.userContactInfoMigrationService.migrate(); -// return true; -// -// } -// -// @GetMapping("users") -// @Transactional -// public boolean migrateUsers() throws JsonProcessingException { -// this.userMigrationService.migrate(); -// return true; -// -// } -// -// -// @GetMapping("dmps") -// @Transactional -// public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException { -// this.dmpMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("datasets") -// @Transactional -// public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { -// this.datasetMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("tags") -// @Transactional -// public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { -// this.tagMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("dmp-dataset-profiles") -// @Transactional -// public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { -// this.dmpDatasetProfileMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("references") -// @Transactional -// public boolean migrateReferences() { -// this.dataRepositoryMigrationService.migrate(); -// this.externalDatasetMigrationService.migrate(); -// this.funderMigrationService.migrate(); -// this.grantMigrationService.migrate(); -// this.organizationMigrationService.migrate(); -// this.projectMigrationService.migrate(); -// this.registryMigrationService.migrate(); -// this.researcherMigrationService.migrate(); -// this.serviceMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("dmp-users") -// @Transactional -// public boolean migrateDmpUsers() { -// this.dmpUserMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("dataset-references") -// @Transactional -// public boolean migrateDatasetReferences() { -// this.referenceMigrationService.migrateDatasetReferences(); -// return true; -// } -// -// @GetMapping("organizations") -// @Transactional -// public boolean migrateOrganizations() { -// this.organizationMigrationService.migrate(); -// return true; -// } -// -// -// @GetMapping("dataRepositories") -// @Transactional -// public boolean migrateDataRepositories() { -// this.dataRepositoryMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("externalDatasets") -// @Transactional -// public boolean migrateExternalDatasets() { -// this.externalDatasetMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("funders") -// @Transactional -// public boolean migrateFunders() { -// this.funderMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("grants") -// @Transactional -// public boolean migrateGrants() { -// this.grantMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("projects") -// @Transactional -// public boolean migrateProjects() { -// this.projectMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("registries") -// @Transactional -// public boolean migrateRegistries() { -// this.registryMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("researchers") -// @Transactional -// public boolean migrateResearchers() { -// this.researcherMigrationService.migrate(); -// return true; -// } -// -// @GetMapping("services") -// @Transactional -// public boolean migrateServices() { -// this.serviceMigrationService.migrate(); -// return true; -// } + @GetMapping("dmp-blueprints-xml") + @Transactional + public boolean migrateDmpBlueprint() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException { + this.dmpBlueprintXmlMigrationService.migrate(); + return true; + + } + + @GetMapping("storage-files") + @Transactional + public boolean migrateStorageFile() throws IOException { + this.storageFileMigrationService.migrate(); + return true; + + } + + @GetMapping("description-template-xml") + @Transactional + public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { + this.descriptionTemplateXmlMigrationService.migrate(); + return true; + + } + + @GetMapping("description-template-xml-clean-invalid-reference-types") + @Transactional + public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { + this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate(); + return true; + + } + + @GetMapping("contacts") + @Transactional + public boolean migrateContacts() { + this.userContactInfoMigrationService.migrate(); + return true; + + } + + @GetMapping("users") + @Transactional + public boolean migrateUsers() throws JsonProcessingException { + this.userMigrationService.migrate(); + return true; + + } + + + @GetMapping("dmps") + @Transactional + public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException { + this.dmpMigrationService.migrate(); + return true; + } + + @GetMapping("datasets") + @Transactional + public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { + this.datasetMigrationService.migrate(); + return true; + } + + @GetMapping("tags") + @Transactional + public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { + this.tagMigrationService.migrate(); + return true; + } + + @GetMapping("dmp-dataset-profiles") + @Transactional + public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { + this.dmpDatasetProfileMigrationService.migrate(); + return true; + } + + @GetMapping("references") + @Transactional + public boolean migrateReferences() { + this.dataRepositoryMigrationService.migrate(); + this.externalDatasetMigrationService.migrate(); + this.funderMigrationService.migrate(); + this.grantMigrationService.migrate(); + this.organizationMigrationService.migrate(); + this.projectMigrationService.migrate(); + this.registryMigrationService.migrate(); + this.researcherMigrationService.migrate(); + this.serviceMigrationService.migrate(); + return true; + } + + @GetMapping("dmp-users") + @Transactional + public boolean migrateDmpUsers() { + this.dmpUserMigrationService.migrate(); + return true; + } + + @GetMapping("dataset-references") + @Transactional + public boolean migrateDatasetReferences() { + this.datasetReferenceMigrationService.migrateDatasetReferences(); + return true; + } + + @GetMapping("organizations") + @Transactional + public boolean migrateOrganizations() { + this.organizationMigrationService.migrate(); + return true; + } + + + @GetMapping("dataRepositories") + @Transactional + public boolean migrateDataRepositories() { + this.dataRepositoryMigrationService.migrate(); + return true; + } + + @GetMapping("externalDatasets") + @Transactional + public boolean migrateExternalDatasets() { + this.externalDatasetMigrationService.migrate(); + return true; + } + + @GetMapping("funders") + @Transactional + public boolean migrateFunders() { + this.funderMigrationService.migrate(); + return true; + } + + @GetMapping("grants") + @Transactional + public boolean migrateGrants() { + this.grantMigrationService.migrate(); + return true; + } + + @GetMapping("projects") + @Transactional + public boolean migrateProjects() { + this.projectMigrationService.migrate(); + return true; + } + + @GetMapping("registries") + @Transactional + public boolean migrateRegistries() { + this.registryMigrationService.migrate(); + return true; + } + + @GetMapping("researchers") + @Transactional + public boolean migrateResearchers() { + this.researcherMigrationService.migrate(); + return true; + } + + @GetMapping("services") + @Transactional + public boolean migrateServices() { + this.serviceMigrationService.migrate(); + return true; + } }