MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

migration changes

This commit is contained in:
Efstratios Giannopoulos 2024-03-12 16:28:07 +02:00
parent b8f14e957e
commit a0554724a8
5 changed files with 243 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dmp-db-scema/updates/00.01.010_Align_DescriptionTemplate_table.sql
View File

@ -54,7 +54,7 @@ BEGIN
) )
); );
UPDATE public."DescriptionTemplate" SET version_status = 2 where status = 0 id in ( UPDATE public."DescriptionTemplate" SET version_status = 2 where status = 0 and id in (
select dt.id from public."DescriptionTemplate" as dt select dt.id from public."DescriptionTemplate" as dt
where dt.version = where dt.version =
( (

1
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -248,7 +248,6 @@ public class DatasetMigrationService {
for (String key : properties.keySet()) { for (String key : properties.keySet()) {
if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) { if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) {
String newKey = key.trim().substring("multiple_".length()); String newKey = key.trim().substring("multiple_".length());
//newKey = newKey.replace("multiple_formats-to-plan-to-store_fa87dc73-d0b6-b707-c632-369f4191a306_", "");//TODO
String[] keyParts = newKey.split("_"); String[] keyParts = newKey.split("_");
if (keyParts.length > 4 && newKey.contains("_multiple_")){ if (keyParts.length > 4 && newKey.contains("_multiple_")){
String[] multiParts = newKey.split("_multiple_"); String[] multiParts = newKey.split("_multiple_");

25
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java
View File

@ -1,5 +1,6 @@
package eu.old.eudat.migration; package eu.old.eudat.migration;
import com.fasterxml.jackson.annotation.JsonValue;
import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.*; import eu.eudat.commons.enums.*;
import eu.eudat.commons.types.descriptiontemplate.*; import eu.eudat.commons.types.descriptiontemplate.*;
@ -10,7 +11,7 @@ import eu.eudat.convention.ConventionService;
import eu.eudat.data.DescriptionTemplateEntity; import eu.eudat.data.DescriptionTemplateEntity;
import eu.eudat.data.ReferenceEntity; import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.ReferenceTypeEntity; import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.file.transformer.enums.FieldDataExternalDatasetType; import eu.eudat.data.converters.enums.DatabaseEnum;
import eu.eudat.model.DescriptionTemplate; import eu.eudat.model.DescriptionTemplate;
import eu.eudat.query.DescriptionTemplateQuery; import eu.eudat.query.DescriptionTemplateQuery;
import eu.old.eudat.logic.utilities.builders.XmlBuilder; import eu.old.eudat.logic.utilities.builders.XmlBuilder;
@ -618,4 +619,26 @@ public class DescriptionTemplateXmlMigrationService {
throw new MyApplicationException("Migrate DescriptionTemplate autocomplete method is invalid " + method); throw new MyApplicationException("Migrate DescriptionTemplate autocomplete method is invalid " + method);
} }
} }
public enum FieldDataExternalDatasetType implements DatabaseEnum<String> {
ReusedDataset("reused_dataset"),
ProducedDataset("produced_dataset"),
Other("other");
private final String value;
FieldDataExternalDatasetType(String value) {
this.value = value;
}
@JsonValue
public String getValue() {
return value;
}
private static final Map<String, FieldDataExternalDatasetType> map = EnumUtils.getEnumValueMap(FieldDataExternalDatasetType.class);
public static FieldDataExternalDatasetType of(String i) {
return map.get(i);
}
}
} }

60
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java
View File

@ -1,13 +1,11 @@
package eu.old.eudat.migration; package eu.old.eudat.migration;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive; import eu.eudat.commons.enums.IsActive;
import eu.eudat.data.DescriptionTagEntity; import eu.eudat.data.DescriptionTagEntity;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.TagEntity; import eu.eudat.data.TagEntity;
import eu.old.eudat.data.dao.entities.DatasetDao; import eu.old.eudat.data.dao.entities.DatasetDao;
import eu.old.eudat.data.entities.Dataset; import eu.old.eudat.data.entities.Dataset;
import eu.old.eudat.elastic.criteria.DatasetCriteria;
import eu.old.eudat.elastic.entities.Tag; import eu.old.eudat.elastic.entities.Tag;
import eu.old.eudat.elastic.repository.DatasetRepository; import eu.old.eudat.elastic.repository.DatasetRepository;
import eu.old.eudat.logic.services.operations.DatabaseRepository; import eu.old.eudat.logic.services.operations.DatabaseRepository;
@ -29,15 +27,13 @@ public class TagMigrationService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(TagMigrationService.class)); private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(TagMigrationService.class));
private final DatabaseRepository databaseRepository; private final DatabaseRepository databaseRepository;
private final QueryFactory queryFactory;
private final DatasetRepository datasetRepository; private final DatasetRepository datasetRepository;
private static final int PageSize = 500; private static final int PageSize = 500;
private static final boolean TestMode = false; private static final boolean TestMode = false;
private final EntityManager entityManager; private final EntityManager entityManager;
public TagMigrationService(DatabaseRepository databaseRepository, QueryFactory queryFactory, DatasetRepository datasetRepository, EntityManager entityManager) { public TagMigrationService(DatabaseRepository databaseRepository, DatasetRepository datasetRepository, EntityManager entityManager) {
this.databaseRepository = databaseRepository; this.databaseRepository = databaseRepository;
this.queryFactory = queryFactory;
this.datasetRepository = datasetRepository; this.datasetRepository = datasetRepository;
this.entityManager = entityManager; this.entityManager = entityManager;
} }
@ -48,8 +44,8 @@ public class TagMigrationService {
logger.debug("Migrate Tags for Dataset Total : " + total); logger.debug("Migrate Tags for Dataset Total : " + total);
int page = 0; int page = 0;
Set<String> savedTagNames = new HashSet<>(); Map<TagKey, UUID> savedTagIdsByName = new HashMap<>();
Map<String, UUID> savedTagIdsByName = new HashMap<>(); HashSet<UUID> existingTagIds = new HashSet<>();
List<Dataset> items; List<Dataset> items;
do { do {
@ -69,27 +65,28 @@ public class TagMigrationService {
boolean tagAlreadyExists; boolean tagAlreadyExists;
if (elasticDataset.getTags() != null && !elasticDataset.getTags().isEmpty()) { if (elasticDataset.getTags() != null && !elasticDataset.getTags().isEmpty()) {
for (Tag tag : elasticDataset.getTags()) { for (Tag tag : elasticDataset.getTags()) {
tagAlreadyExists = savedTagNames.contains(tag.getName()); //TODO maybe should check owner to recreate tagAlreadyExists = savedTagIdsByName.containsKey(new TagKey(item, tag)); //TODO we want owner logic ?
savedTagNames.add(tag.getName());
if (!tagAlreadyExists) { if (!tagAlreadyExists) {
TagEntity tagEntity = new TagEntity(); TagEntity tagEntity = new TagEntity();
tagEntity.setId(UUID.fromString(tag.getId())); if (!existingTagIds.contains(UUID.fromString(tag.getId()))) tagEntity.setId(UUID.fromString(tag.getId()));
else tagEntity.setId(UUID.randomUUID());
tagEntity.setLabel(tag.getName()); tagEntity.setLabel(tag.getName());
tagEntity.setCreatedAt(Instant.now()); tagEntity.setCreatedAt(Instant.now());
tagEntity.setUpdatedAt(Instant.now()); tagEntity.setUpdatedAt(Instant.now());
if (item.getCreator() != null) tagEntity.setCreatedById(item.getCreator().getId()); if (item.getCreator() != null) tagEntity.setCreatedById(item.getCreator().getId());
tagEntity.setIsActive(IsActive.Active); tagEntity.setIsActive(IsActive.Active);
savedTagIdsByName.put(tag.getName(), tagEntity.getId()); savedTagIdsByName.put(new TagKey(item, tag), tagEntity.getId());
this.entityManager.persist(tagEntity); this.entityManager.persist(tagEntity);
} }
DescriptionTagEntity descriptionTagEntity = new DescriptionTagEntity(); DescriptionTagEntity descriptionTagEntity = new DescriptionTagEntity();
descriptionTagEntity.setId(UUID.randomUUID()); descriptionTagEntity.setId(UUID.randomUUID());
descriptionTagEntity.setTagId(savedTagIdsByName.get(tag.getName())); descriptionTagEntity.setTagId(savedTagIdsByName.get(new TagKey(item, tag)));
descriptionTagEntity.setDescriptionId(item.getId()); descriptionTagEntity.setDescriptionId(item.getId());
descriptionTagEntity.setCreatedAt(Instant.now()); descriptionTagEntity.setCreatedAt(Instant.now());
descriptionTagEntity.setUpdatedAt(Instant.now()); descriptionTagEntity.setUpdatedAt(Instant.now());
descriptionTagEntity.setIsActive(IsActive.Active); descriptionTagEntity.setIsActive(IsActive.Active);
this.entityManager.persist(descriptionTagEntity); this.entityManager.persist(descriptionTagEntity);
existingTagIds.add(descriptionTagEntity.getTagId());
} }
} }
this.entityManager.flush(); this.entityManager.flush();
@ -100,4 +97,41 @@ public class TagMigrationService {
} while (items != null && !items.isEmpty() && !TestMode); } while (items != null && !items.isEmpty() && !TestMode);
} }
public static class TagKey {
private final UUID owner;
private final String name;
private final int hashCode;
public TagKey(Dataset item, Tag tag) {
this.name = tag.getName();
if (item.getCreator() != null) this.owner = item.getCreator().getId();
else this.owner = null;
hashCode = Objects.hash(this.owner, this.name);
}
public UUID getOwner() {
return owner;
}
public String getName() {
return name;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
TagKey that = (TagKey) o;
return Objects.equals(owner, that.getOwner()) && Objects.equals(name, that.getName());
}
@Override
public int hashCode() {
return this.hashCode;
}
}
} }

339
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -1,5 +1,6 @@
package eu.old.eudat.publicapi.migration; package eu.old.eudat.publicapi.migration;
import com.fasterxml.jackson.core.JsonProcessingException;
import eu.old.eudat.migration.*; import eu.old.eudat.migration.*;
import io.swagger.annotations.Api; import io.swagger.annotations.Api;
import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.JAXBException;
@ -124,174 +125,174 @@ public class MigrationController {
return true; return true;
} }
// @GetMapping("dmp-blueprints-xml") @GetMapping("dmp-blueprints-xml")
// @Transactional @Transactional
// public boolean migrateDmpBlueprint() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException { public boolean migrateDmpBlueprint() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException {
// this.dmpBlueprintXmlMigrationService.migrate(); this.dmpBlueprintXmlMigrationService.migrate();
// return true; return true;
//
// } }
//
// @GetMapping("storage-files") @GetMapping("storage-files")
// @Transactional @Transactional
// public boolean migrateStorageFile() throws IOException { public boolean migrateStorageFile() throws IOException {
// this.storageFileMigrationService.migrate(); this.storageFileMigrationService.migrate();
// return true; return true;
//
// } }
//
// @GetMapping("description-template-xml") @GetMapping("description-template-xml")
// @Transactional @Transactional
// public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
// this.descriptionTemplateXmlMigrationService.migrate(); this.descriptionTemplateXmlMigrationService.migrate();
// return true; return true;
//
// } }
//
// @GetMapping("description-template-xml-clean-invalid-reference-types") @GetMapping("description-template-xml-clean-invalid-reference-types")
// @Transactional @Transactional
// public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
// this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate(); this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate();
// return true; return true;
//
// } }
//
// @GetMapping("contacts") @GetMapping("contacts")
// @Transactional @Transactional
// public boolean migrateContacts() { public boolean migrateContacts() {
// this.userContactInfoMigrationService.migrate(); this.userContactInfoMigrationService.migrate();
// return true; return true;
//
// } }
//
// @GetMapping("users") @GetMapping("users")
// @Transactional @Transactional
// public boolean migrateUsers() throws JsonProcessingException { public boolean migrateUsers() throws JsonProcessingException {
// this.userMigrationService.migrate(); this.userMigrationService.migrate();
// return true; return true;
//
// } }
//
//
// @GetMapping("dmps") @GetMapping("dmps")
// @Transactional @Transactional
// public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException { public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException {
// this.dmpMigrationService.migrate(); this.dmpMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("datasets") @GetMapping("datasets")
// @Transactional @Transactional
// public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
// this.datasetMigrationService.migrate(); this.datasetMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("tags") @GetMapping("tags")
// @Transactional @Transactional
// public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
// this.tagMigrationService.migrate(); this.tagMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("dmp-dataset-profiles") @GetMapping("dmp-dataset-profiles")
// @Transactional @Transactional
// public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
// this.dmpDatasetProfileMigrationService.migrate(); this.dmpDatasetProfileMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("references") @GetMapping("references")
// @Transactional @Transactional
// public boolean migrateReferences() { public boolean migrateReferences() {
// this.dataRepositoryMigrationService.migrate(); this.dataRepositoryMigrationService.migrate();
// this.externalDatasetMigrationService.migrate(); this.externalDatasetMigrationService.migrate();
// this.funderMigrationService.migrate(); this.funderMigrationService.migrate();
// this.grantMigrationService.migrate(); this.grantMigrationService.migrate();
// this.organizationMigrationService.migrate(); this.organizationMigrationService.migrate();
// this.projectMigrationService.migrate(); this.projectMigrationService.migrate();
// this.registryMigrationService.migrate(); this.registryMigrationService.migrate();
// this.researcherMigrationService.migrate(); this.researcherMigrationService.migrate();
// this.serviceMigrationService.migrate(); this.serviceMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("dmp-users") @GetMapping("dmp-users")
// @Transactional @Transactional
// public boolean migrateDmpUsers() { public boolean migrateDmpUsers() {
// this.dmpUserMigrationService.migrate(); this.dmpUserMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("dataset-references") @GetMapping("dataset-references")
// @Transactional @Transactional
// public boolean migrateDatasetReferences() { public boolean migrateDatasetReferences() {
// this.referenceMigrationService.migrateDatasetReferences(); this.datasetReferenceMigrationService.migrateDatasetReferences();
// return true; return true;
// } }
//
// @GetMapping("organizations") @GetMapping("organizations")
// @Transactional @Transactional
// public boolean migrateOrganizations() { public boolean migrateOrganizations() {
// this.organizationMigrationService.migrate(); this.organizationMigrationService.migrate();
// return true; return true;
// } }
//
//
// @GetMapping("dataRepositories") @GetMapping("dataRepositories")
// @Transactional @Transactional
// public boolean migrateDataRepositories() { public boolean migrateDataRepositories() {
// this.dataRepositoryMigrationService.migrate(); this.dataRepositoryMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("externalDatasets") @GetMapping("externalDatasets")
// @Transactional @Transactional
// public boolean migrateExternalDatasets() { public boolean migrateExternalDatasets() {
// this.externalDatasetMigrationService.migrate(); this.externalDatasetMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("funders") @GetMapping("funders")
// @Transactional @Transactional
// public boolean migrateFunders() { public boolean migrateFunders() {
// this.funderMigrationService.migrate(); this.funderMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("grants") @GetMapping("grants")
// @Transactional @Transactional
// public boolean migrateGrants() { public boolean migrateGrants() {
// this.grantMigrationService.migrate(); this.grantMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("projects") @GetMapping("projects")
// @Transactional @Transactional
// public boolean migrateProjects() { public boolean migrateProjects() {
// this.projectMigrationService.migrate(); this.projectMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("registries") @GetMapping("registries")
// @Transactional @Transactional
// public boolean migrateRegistries() { public boolean migrateRegistries() {
// this.registryMigrationService.migrate(); this.registryMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("researchers") @GetMapping("researchers")
// @Transactional @Transactional
// public boolean migrateResearchers() { public boolean migrateResearchers() {
// this.researcherMigrationService.migrate(); this.researcherMigrationService.migrate();
// return true; return true;
// } }
//
// @GetMapping("services") @GetMapping("services")
// @Transactional @Transactional
// public boolean migrateServices() { public boolean migrateServices() {
// this.serviceMigrationService.migrate(); this.serviceMigrationService.migrate();
// return true; return true;
// } }
} }