MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

migration changes

This commit is contained in:
Efstratios Giannopoulos 2024-03-12 16:28:07 +02:00
parent b8f14e957e
commit a0554724a8
5 changed files with 243 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dmp-db-scema/updates/00.01.010_Align_DescriptionTemplate_table.sql
View File

@ -54,7 +54,7 @@ BEGIN
)
);
UPDATE public."DescriptionTemplate" SET version_status = 2 where status = 0 id in (
UPDATE public."DescriptionTemplate" SET version_status = 2 where status = 0 and id in (
select dt.id from public."DescriptionTemplate" as dt
where dt.version =
(

1
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -248,7 +248,6 @@ public class DatasetMigrationService {
for (String key : properties.keySet()) {
if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) {
String newKey = key.trim().substring("multiple_".length());
//newKey = newKey.replace("multiple_formats-to-plan-to-store_fa87dc73-d0b6-b707-c632-369f4191a306_", "");//TODO
String[] keyParts = newKey.split("_");
if (keyParts.length > 4 && newKey.contains("_multiple_")){
String[] multiParts = newKey.split("_multiple_");

25
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java
View File

@ -1,5 +1,6 @@
package eu.old.eudat.migration;
import com.fasterxml.jackson.annotation.JsonValue;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.*;
import eu.eudat.commons.types.descriptiontemplate.*;
@ -10,7 +11,7 @@ import eu.eudat.convention.ConventionService;
import eu.eudat.data.DescriptionTemplateEntity;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.file.transformer.enums.FieldDataExternalDatasetType;
import eu.eudat.data.converters.enums.DatabaseEnum;
import eu.eudat.model.DescriptionTemplate;
import eu.eudat.query.DescriptionTemplateQuery;
import eu.old.eudat.logic.utilities.builders.XmlBuilder;
@ -618,4 +619,26 @@ public class DescriptionTemplateXmlMigrationService {
throw new MyApplicationException("Migrate DescriptionTemplate autocomplete method is invalid " + method);
}
}
public enum FieldDataExternalDatasetType implements DatabaseEnum<String> {
ReusedDataset("reused_dataset"),
ProducedDataset("produced_dataset"),
Other("other");
private final String value;
FieldDataExternalDatasetType(String value) {
this.value = value;
}
@JsonValue
public String getValue() {
return value;
}
private static final Map<String, FieldDataExternalDatasetType> map = EnumUtils.getEnumValueMap(FieldDataExternalDatasetType.class);
public static FieldDataExternalDatasetType of(String i) {
return map.get(i);
}
}
}

60
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java
View File

@ -1,13 +1,11 @@
package eu.old.eudat.migration;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.data.DescriptionTagEntity;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.TagEntity;
import eu.old.eudat.data.dao.entities.DatasetDao;
import eu.old.eudat.data.entities.Dataset;
import eu.old.eudat.elastic.criteria.DatasetCriteria;
import eu.old.eudat.elastic.entities.Tag;
import eu.old.eudat.elastic.repository.DatasetRepository;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
@ -29,15 +27,13 @@ public class TagMigrationService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(TagMigrationService.class));
private final DatabaseRepository databaseRepository;
private final QueryFactory queryFactory;
private final DatasetRepository datasetRepository;
private static final int PageSize = 500;
private static final boolean TestMode = false;
private final EntityManager entityManager;
public TagMigrationService(DatabaseRepository databaseRepository, QueryFactory queryFactory, DatasetRepository datasetRepository, EntityManager entityManager) {
public TagMigrationService(DatabaseRepository databaseRepository, DatasetRepository datasetRepository, EntityManager entityManager) {
this.databaseRepository = databaseRepository;
this.queryFactory = queryFactory;
this.datasetRepository = datasetRepository;
this.entityManager = entityManager;
}
@ -48,8 +44,8 @@ public class TagMigrationService {
logger.debug("Migrate Tags for Dataset Total : " + total);
int page = 0;
Set<String> savedTagNames = new HashSet<>();
Map<String, UUID> savedTagIdsByName = new HashMap<>();
Map<TagKey, UUID> savedTagIdsByName = new HashMap<>();
HashSet<UUID> existingTagIds = new HashSet<>();
List<Dataset> items;
do {
@ -69,27 +65,28 @@ public class TagMigrationService {
boolean tagAlreadyExists;
if (elasticDataset.getTags() != null && !elasticDataset.getTags().isEmpty()) {
for (Tag tag : elasticDataset.getTags()) {
tagAlreadyExists = savedTagNames.contains(tag.getName()); //TODO maybe should check owner to recreate
savedTagNames.add(tag.getName());
tagAlreadyExists = savedTagIdsByName.containsKey(new TagKey(item, tag)); //TODO we want owner logic ?
if (!tagAlreadyExists) {
TagEntity tagEntity = new TagEntity();
tagEntity.setId(UUID.fromString(tag.getId()));
if (!existingTagIds.contains(UUID.fromString(tag.getId()))) tagEntity.setId(UUID.fromString(tag.getId()));
else tagEntity.setId(UUID.randomUUID());
tagEntity.setLabel(tag.getName());
tagEntity.setCreatedAt(Instant.now());
tagEntity.setUpdatedAt(Instant.now());
if (item.getCreator() != null) tagEntity.setCreatedById(item.getCreator().getId());
tagEntity.setIsActive(IsActive.Active);
savedTagIdsByName.put(tag.getName(), tagEntity.getId());
savedTagIdsByName.put(new TagKey(item, tag), tagEntity.getId());
this.entityManager.persist(tagEntity);
}
DescriptionTagEntity descriptionTagEntity = new DescriptionTagEntity();
descriptionTagEntity.setId(UUID.randomUUID());
descriptionTagEntity.setTagId(savedTagIdsByName.get(tag.getName()));
descriptionTagEntity.setTagId(savedTagIdsByName.get(new TagKey(item, tag)));
descriptionTagEntity.setDescriptionId(item.getId());
descriptionTagEntity.setCreatedAt(Instant.now());
descriptionTagEntity.setUpdatedAt(Instant.now());
descriptionTagEntity.setIsActive(IsActive.Active);
this.entityManager.persist(descriptionTagEntity);
existingTagIds.add(descriptionTagEntity.getTagId());
}
}
this.entityManager.flush();
@ -100,4 +97,41 @@ public class TagMigrationService {
} while (items != null && !items.isEmpty() && !TestMode);
}
public static class TagKey {
private final UUID owner;
private final String name;
private final int hashCode;
public TagKey(Dataset item, Tag tag) {
this.name = tag.getName();
if (item.getCreator() != null) this.owner = item.getCreator().getId();
else this.owner = null;
hashCode = Objects.hash(this.owner, this.name);
}
public UUID getOwner() {
return owner;
}
public String getName() {
return name;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
TagKey that = (TagKey) o;
return Objects.equals(owner, that.getOwner()) && Objects.equals(name, that.getName());
}
@Override
public int hashCode() {
return this.hashCode;
}
}
}

341
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -1,5 +1,6 @@
package eu.old.eudat.publicapi.migration;
import com.fasterxml.jackson.core.JsonProcessingException;
import eu.old.eudat.migration.*;
import io.swagger.annotations.Api;
import jakarta.xml.bind.JAXBException;
@ -112,7 +113,7 @@ public class MigrationController {
this.dmpMigrationService.migrate();
this.dmpDatasetProfileMigrationService.migrate();
this.dmpUserMigrationService.migrate();
//Description
this.datasetMigrationService.migrate();
this.datasetReferenceMigrationService.migrateDatasetReferences();
@ -124,174 +125,174 @@ public class MigrationController {
return true;
}
// @GetMapping("dmp-blueprints-xml")
// @Transactional
// public boolean migrateDmpBlueprint() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException {
// this.dmpBlueprintXmlMigrationService.migrate();
// return true;
//
// }
//
// @GetMapping("storage-files")
// @Transactional
// public boolean migrateStorageFile() throws IOException {
// this.storageFileMigrationService.migrate();
// return true;
//
// }
//
// @GetMapping("description-template-xml")
// @Transactional
// public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
// this.descriptionTemplateXmlMigrationService.migrate();
// return true;
//
// }
//
// @GetMapping("description-template-xml-clean-invalid-reference-types")
// @Transactional
// public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
// this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate();
// return true;
//
// }
//
// @GetMapping("contacts")
// @Transactional
// public boolean migrateContacts() {
// this.userContactInfoMigrationService.migrate();
// return true;
//
// }
//
// @GetMapping("users")
// @Transactional
// public boolean migrateUsers() throws JsonProcessingException {
// this.userMigrationService.migrate();
// return true;
//
// }
//
//
// @GetMapping("dmps")
// @Transactional
// public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException {
// this.dmpMigrationService.migrate();
// return true;
// }
//
// @GetMapping("datasets")
// @Transactional
// public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
// this.datasetMigrationService.migrate();
// return true;
// }
//
// @GetMapping("tags")
// @Transactional
// public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
// this.tagMigrationService.migrate();
// return true;
// }
//
// @GetMapping("dmp-dataset-profiles")
// @Transactional
// public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
// this.dmpDatasetProfileMigrationService.migrate();
// return true;
// }
//
// @GetMapping("references")
// @Transactional
// public boolean migrateReferences() {
// this.dataRepositoryMigrationService.migrate();
// this.externalDatasetMigrationService.migrate();
// this.funderMigrationService.migrate();
// this.grantMigrationService.migrate();
// this.organizationMigrationService.migrate();
// this.projectMigrationService.migrate();
// this.registryMigrationService.migrate();
// this.researcherMigrationService.migrate();
// this.serviceMigrationService.migrate();
// return true;
// }
//
// @GetMapping("dmp-users")
// @Transactional
// public boolean migrateDmpUsers() {
// this.dmpUserMigrationService.migrate();
// return true;
// }
//
// @GetMapping("dataset-references")
// @Transactional
// public boolean migrateDatasetReferences() {
// this.referenceMigrationService.migrateDatasetReferences();
// return true;
// }
//
// @GetMapping("organizations")
// @Transactional
// public boolean migrateOrganizations() {
// this.organizationMigrationService.migrate();
// return true;
// }
//
//
// @GetMapping("dataRepositories")
// @Transactional
// public boolean migrateDataRepositories() {
// this.dataRepositoryMigrationService.migrate();
// return true;
// }
//
// @GetMapping("externalDatasets")
// @Transactional
// public boolean migrateExternalDatasets() {
// this.externalDatasetMigrationService.migrate();
// return true;
// }
//
// @GetMapping("funders")
// @Transactional
// public boolean migrateFunders() {
// this.funderMigrationService.migrate();
// return true;
// }
//
// @GetMapping("grants")
// @Transactional
// public boolean migrateGrants() {
// this.grantMigrationService.migrate();
// return true;
// }
//
// @GetMapping("projects")
// @Transactional
// public boolean migrateProjects() {
// this.projectMigrationService.migrate();
// return true;
// }
//
// @GetMapping("registries")
// @Transactional
// public boolean migrateRegistries() {
// this.registryMigrationService.migrate();
// return true;
// }
//
// @GetMapping("researchers")
// @Transactional
// public boolean migrateResearchers() {
// this.researcherMigrationService.migrate();
// return true;
// }
//
// @GetMapping("services")
// @Transactional
// public boolean migrateServices() {
// this.serviceMigrationService.migrate();
// return true;
// }
@GetMapping("dmp-blueprints-xml")
@Transactional
public boolean migrateDmpBlueprint() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException {
this.dmpBlueprintXmlMigrationService.migrate();
return true;
}
@GetMapping("storage-files")
@Transactional
public boolean migrateStorageFile() throws IOException {
this.storageFileMigrationService.migrate();
return true;
}
@GetMapping("description-template-xml")
@Transactional
public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
this.descriptionTemplateXmlMigrationService.migrate();
return true;
}
@GetMapping("description-template-xml-clean-invalid-reference-types")
@Transactional
public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate();
return true;
}
@GetMapping("contacts")
@Transactional
public boolean migrateContacts() {
this.userContactInfoMigrationService.migrate();
return true;
}
@GetMapping("users")
@Transactional
public boolean migrateUsers() throws JsonProcessingException {
this.userMigrationService.migrate();
return true;
}
@GetMapping("dmps")
@Transactional
public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException {
this.dmpMigrationService.migrate();
return true;
}
@GetMapping("datasets")
@Transactional
public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
this.datasetMigrationService.migrate();
return true;
}
@GetMapping("tags")
@Transactional
public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
this.tagMigrationService.migrate();
return true;
}
@GetMapping("dmp-dataset-profiles")
@Transactional
public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
this.dmpDatasetProfileMigrationService.migrate();
return true;
}
@GetMapping("references")
@Transactional
public boolean migrateReferences() {
this.dataRepositoryMigrationService.migrate();
this.externalDatasetMigrationService.migrate();
this.funderMigrationService.migrate();
this.grantMigrationService.migrate();
this.organizationMigrationService.migrate();
this.projectMigrationService.migrate();
this.registryMigrationService.migrate();
this.researcherMigrationService.migrate();
this.serviceMigrationService.migrate();
return true;
}
@GetMapping("dmp-users")
@Transactional
public boolean migrateDmpUsers() {
this.dmpUserMigrationService.migrate();
return true;
}
@GetMapping("dataset-references")
@Transactional
public boolean migrateDatasetReferences() {
this.datasetReferenceMigrationService.migrateDatasetReferences();
return true;
}
@GetMapping("organizations")
@Transactional
public boolean migrateOrganizations() {
this.organizationMigrationService.migrate();
return true;
}
@GetMapping("dataRepositories")
@Transactional
public boolean migrateDataRepositories() {
this.dataRepositoryMigrationService.migrate();
return true;
}
@GetMapping("externalDatasets")
@Transactional
public boolean migrateExternalDatasets() {
this.externalDatasetMigrationService.migrate();
return true;
}
@GetMapping("funders")
@Transactional
public boolean migrateFunders() {
this.funderMigrationService.migrate();
return true;
}
@GetMapping("grants")
@Transactional
public boolean migrateGrants() {
this.grantMigrationService.migrate();
return true;
}
@GetMapping("projects")
@Transactional
public boolean migrateProjects() {
this.projectMigrationService.migrate();
return true;
}
@GetMapping("registries")
@Transactional
public boolean migrateRegistries() {
this.registryMigrationService.migrate();
return true;
}
@GetMapping("researchers")
@Transactional
public boolean migrateResearchers() {
this.researcherMigrationService.migrate();
return true;
}
@GetMapping("services")
@Transactional
public boolean migrateServices() {
this.serviceMigrationService.migrate();
return true;
}
}