diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/externalfetcher/ExternalFetcherServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/externalfetcher/ExternalFetcherServiceImpl.java index 03c550be8..419b41d38 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/externalfetcher/ExternalFetcherServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/externalfetcher/ExternalFetcherServiceImpl.java @@ -15,6 +15,9 @@ import eu.eudat.service.externalfetcher.config.entities.*; import eu.eudat.service.externalfetcher.models.ExternalDataResult; import eu.eudat.service.externalfetcher.criteria.ExternalReferenceCriteria; import gr.cite.tools.exception.MyApplicationException; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import net.minidev.json.JSONArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +30,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.reactive.function.client.WebClient; import reactor.netty.http.client.HttpClient; +import javax.net.ssl.SSLException; import java.util.*; import java.util.stream.Collectors; @@ -43,7 +47,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService { this.jsonHandlingService = jsonHandlingService; } - private WebClient getWebClient(){ + private WebClient getWebClient() { if (this.webClient == null) { this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> { clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON)); @@ -60,7 +64,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService { public ExternalDataResult getExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key) { List apiSourcesToUse = sources; if (!this.conventionService.isNullOrEmpty(key)){ - apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).toList(); + apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).collect(Collectors.toList()); } if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult(); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DataRepositoryMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DataRepositoryMigrationService.java index 7f27ccc05..c6f787a16 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DataRepositoryMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DataRepositoryMigrationService.java @@ -53,7 +53,7 @@ public class DataRepositoryMigrationService { entityManager.detach(item); if (item.getReference() == null || !item.getReference().contains(":")){ logger.warn("Reference generated because is null DataRepository " + item.getId()); - item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); } eu.old.eudat.models.data.datarepository.DataRepositoryModel model = new eu.old.eudat.models.data.datarepository.DataRepositoryModel().fromDataModel(item); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java index f197cefb1..09fb7fab8 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java @@ -2,15 +2,12 @@ package eu.old.eudat.migration; import eu.eudat.commons.JsonHandlingService; import eu.eudat.commons.XmlHandlingService; -import eu.eudat.commons.enums.DescriptionStatus; -import eu.eudat.commons.enums.IsActive; -import eu.eudat.commons.enums.ReferenceFieldDataType; -import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.*; import eu.eudat.commons.types.description.*; import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity; import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity; import eu.eudat.commons.types.dmpblueprint.DefinitionEntity; -import eu.eudat.commons.types.externalfetcher.ExternalFetcherBaseSourceConfigurationEntity; +import eu.eudat.commons.types.externalfetcher.ExternalFetcherApiSourceConfigurationEntity; import eu.eudat.commons.types.referencetype.ReferenceTypeDefinitionEntity; import eu.eudat.convention.ConventionService; import eu.eudat.data.*; @@ -18,16 +15,17 @@ import eu.eudat.model.Dmp; import eu.eudat.model.Reference; import eu.eudat.model.ReferenceType; import eu.eudat.query.*; +import eu.eudat.service.reference.ReferenceService; import eu.old.eudat.data.dao.entities.DatasetDao; import eu.old.eudat.data.entities.Dataset; import eu.old.eudat.logic.services.operations.DatabaseRepository; -import eu.old.eudat.models.data.user.components.datasetprofile.Field; import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.fieldset.BaseFieldSet; import gr.cite.tools.logging.LoggerService; import jakarta.persistence.EntityManager; import jakarta.xml.bind.JAXBException; +import org.apache.commons.lang3.StringEscapeUtils; import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -40,6 +38,8 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; @Service @@ -54,14 +54,16 @@ public class DatasetMigrationService { private static final boolean TestMode = false; private final EntityManager entityManager; private final ConventionService conventionService; + private final ReferenceService referenceService; - public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager, ConventionService conventionService) { + public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager, ConventionService conventionService, ReferenceService referenceService) { this.databaseRepository = databaseRepository; this.jsonHandlingService = jsonHandlingService; this.queryFactory = queryFactory; this.xmlHandlingService = xmlHandlingService; this.entityManager = entityManager; this.conventionService = conventionService; + this.referenceService = referenceService; } public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { @@ -97,7 +99,7 @@ public class DatasetMigrationService { List dmpDescriptionTemplateEntities = this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collect(); List descriptionTemplates = this.queryFactory.query(DescriptionTemplateQuery.class).ids(items.stream().map(x-> x.getProfile().getId()).distinct().toList()).collect(); - Map creatorsByDmp = this.queryFactory.query(DmpQuery.class).ids(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collectAs(new BaseFieldSet().ensure(Dmp._id).ensure(Dmp._creator)).stream().collect(Collectors.toMap(DmpEntity::getId, DmpEntity::getCreatorId));; + Map creatorsByDmp = this.queryFactory.query(DmpQuery.class).ids(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collectAs(new BaseFieldSet().ensure(Dmp._id).ensure(Dmp._creator)).stream().collect(Collectors.toMap(DmpEntity::getId, DmpEntity::getCreatorId)); Map descriptionTemplateDefinitionMap = new HashMap<>(); for (DescriptionTemplateEntity descriptionTemplateEntity : descriptionTemplates){ @@ -107,20 +109,17 @@ public class DatasetMigrationService { DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null); if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) { - logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); } UUID sectionId = definition.getSections().get(item.getDmpSectionIndex()).getId(); if (sectionId == null) { - logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); } List itemDescriptionTemplates = this.getOrCreateDmpDescriptionTemplateEntity(item, sectionId, dmpDescriptionTemplateEntities); if (itemDescriptionTemplates.size() > 1) { - logger.error("Migrate Dataset " + item.getId() + " multiple DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); - //TODO CLEAN Duplicates dmp, DescriptionTemplate, SectionId + throw new MyApplicationException("Migrate Dataset " + item.getId() + " multiple DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); } DescriptionEntity data = new DescriptionEntity(); @@ -151,7 +150,6 @@ public class DatasetMigrationService { data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap))); if (data.getCreatedById() == null){ - logger.error("Migration skipped creator not found " + item.getId()); throw new MyApplicationException("Migration skipped creator not found " + item.getId()); } @@ -162,13 +160,12 @@ public class DatasetMigrationService { page++; } } while (items != null && !items.isEmpty() && !TestMode); - throw new MyApplicationException(""); } private List getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List dmpDescriptionTemplateEntities){ List itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList(); if (itemDescriptionTemplates.isEmpty()) { - logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); + logger.warn("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); if (dmpDescriptionTemplateEntities.stream().anyMatch(x -> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId))) { DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity(); dmpDescriptionTemplateEntity.setId(UUID.randomUUID()); @@ -181,7 +178,7 @@ public class DatasetMigrationService { this.entityManager.persist(dmpDescriptionTemplateEntity); itemDescriptionTemplates = List.of(dmpDescriptionTemplateEntity); } else { - throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); + throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getLabel() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); } } return itemDescriptionTemplates; @@ -203,7 +200,7 @@ public class DatasetMigrationService { FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().equals(key.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null); if (currentFieldSet == null) { - logger.error("Field set " + key + " not found for comment " + properties.get(key).toString()); + logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set " + key + " not found for comment " + properties.get(key).toString()); continue; } @@ -221,16 +218,16 @@ public class DatasetMigrationService { } if (currentField == null) { - logger.error("Field set not found for field " + key + " " + properties.get(key).toString()); + logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set not found for field " + key + " " + properties.get(key).toString()); continue; } - this.addSimpleField(propertyDefinitionEntity, currentFieldSet, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap); + this.addSimpleField(item, propertyDefinitionEntity, currentFieldSet, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap); } } } - this.addMultipleField(properties, descriptionTemplateDefinitionEntity, propertyDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap); + this.addMultipleField(item, properties, descriptionTemplateDefinitionEntity, propertyDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap); for (PropertyDefinitionFieldSetEntity fieldSetEntity : propertyDefinitionEntity.getFieldSets().values()){ int newOrdinal = 0; @@ -246,39 +243,45 @@ public class DatasetMigrationService { return propertyDefinitionEntity; } - private void addMultipleField(Map properties, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, PropertyDefinitionEntity propertyDefinitionEntity, Map referenceTypeDefinitionEntityMap, Map referenceMap){ + private void addMultipleField(Dataset item, Map properties, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, PropertyDefinitionEntity propertyDefinitionEntity, Map referenceTypeDefinitionEntityMap, Map referenceMap){ Map groupOrdinalMapping = new HashMap<>(); for (String key : properties.keySet()) { if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) { String newKey = key.trim().substring("multiple_".length()); - String[] keyParts = newKey.split("_", 4); + //newKey = newKey.replace("multiple_formats-to-plan-to-store_fa87dc73-d0b6-b707-c632-369f4191a306_", "");//TODO + String[] keyParts = newKey.split("_"); + if (keyParts.length > 4 && newKey.contains("_multiple_")){ + String[] multiParts = newKey.split("_multiple_"); + if (multiParts.length == 2){ + newKey = multiParts[0] + "_" + multiParts[1].split("_")[multiParts[1].split("_").length - 1]; + keyParts = newKey.split("_"); + } + } String fieldSetId = ""; String groupId = ""; int ordinal = Integer.MIN_VALUE; String fieldId = ""; - if (keyParts.length == 4) { - fieldSetId = keyParts[0].trim(); - groupId = keyParts[1].trim(); - fieldId = keyParts[3].trim(); - } else if (keyParts.length == 3) { - fieldSetId = keyParts[0].trim(); - groupId = keyParts[1].trim(); - fieldId = keyParts[2].trim(); - } else { - logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString()); - continue; - } + if (keyParts.length == 4) { + fieldSetId = keyParts[0].trim(); + groupId = keyParts[1].trim(); + fieldId = keyParts[3].trim(); + } else if (keyParts.length == 3) { + fieldSetId = keyParts[0].trim(); + groupId = keyParts[1].trim(); + fieldId = keyParts[2].trim(); + } else { + throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field group key has invalid format " + key + " " + properties.get(key).toString()); + } try { if (keyParts.length == 4) ordinal = Integer.parseInt(keyParts[2].trim()); } catch (Exception e){ - logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString()); - continue; + throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field group key has invalid format " + key + " " + properties.get(key).toString()); } eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = descriptionTemplateDefinitionEntity.getFieldById(fieldId).stream().findFirst().orElse(null); if (currentField == null) { - logger.error("Field set not found for field " + key + " " + properties.get(key).toString()); + logger.warn("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set not found for field " + key + " " + properties.get(key).toString()); continue; } @@ -303,13 +306,13 @@ public class DatasetMigrationService { if (groupOrdinalMapping.get(new MultipleFieldKey(fieldSetId, groupId)) != ordinal) throw new MyApplicationException("Invalid multiple key ordinal " + key); } - this.addMultipleField(propertyDefinitionFieldSetEntity, ordinal, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap); + this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap); } } } - private void addMultipleField(PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, Map referenceTypeDefinitionEntityMap, Map referenceMap){ + private void addMultipleField(Dataset item, PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, Map referenceTypeDefinitionEntityMap, Map referenceMap){ PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null); if (propertyDefinitionFieldSetItemEntity == null){ @@ -319,10 +322,10 @@ public class DatasetMigrationService { propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); } - propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); + propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); } - private void addSimpleField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, Map referenceTypeDefinitionEntityMap, Map referenceMap){ + private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, Map referenceTypeDefinitionEntityMap, Map referenceMap){ PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null); if (propertyDefinitionFieldSetEntity == null) { propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); @@ -340,7 +343,7 @@ public class DatasetMigrationService { propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst(); } - propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); + propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); } @@ -365,7 +368,7 @@ public class DatasetMigrationService { propertyDefinitionFieldSetItemEntity.setComment(comment); } - private FieldEntity buildField(eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, Map referenceTypeDefinitionEntityMap, Map referenceMap){ + private FieldEntity buildField(Dataset item, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, Map referenceTypeDefinitionEntityMap, Map referenceMap){ FieldEntity fieldEntity = new FieldEntity(); String textValue = properties.get(currentField.getId()) != null ? properties.get(currentField.getId()).toString() : null; if (textValue == null || textValue.isEmpty()) return fieldEntity; @@ -385,26 +388,23 @@ public class DatasetMigrationService { } case SELECT -> { if(!this.conventionService.isNullOrEmpty(textValue)) { - String[] valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, textValue); - if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, this.cleanAsObjectString(textValue)); + String[] valuesParsed = this.tryParseJsonAsObjectString(String[].class, textValue); fieldEntity.setTextListValue(valuesParsed == null ? List.of(textValue) : Arrays.stream(valuesParsed).toList()); } } case DATASET_IDENTIFIER, VALIDATION -> { if(!this.conventionService.isNullOrEmpty(textValue)) { - ExternalIdentifierEntity externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, textValue); - if (externalIdentifierEntity == null) externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, this.cleanAsObjectString(textValue)); + ExternalIdentifierEntity externalIdentifierEntity = this.tryParseJsonAsObjectString(ExternalIdentifierEntity.class, textValue); if (externalIdentifierEntity == null) throw new MyApplicationException("Could not parse dataset External Identifier : " + textValue); fieldEntity.setExternalIdentifier(externalIdentifierEntity); } } case UPLOAD -> { if(!this.conventionService.isNullOrEmpty(textValue)) { - Map valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, textValue); + Map valuesParsed = this.tryParseJsonAsObjectString(Map.class, textValue); if (valuesParsed == null) { valuesParsed = (Map) properties.get(currentField.getId()); } - if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, this.cleanAsObjectString(textValue)); if (valuesParsed == null) throw new MyApplicationException("Could not parse upload : " + textValue); String id = valuesParsed.getOrDefault("id", null); if (id == null) throw new MyApplicationException("Could not parse upload : " + textValue); @@ -413,8 +413,7 @@ public class DatasetMigrationService { } case CURRENCY -> { if(!this.conventionService.isNullOrEmpty(textValue)) { - Currency currency = this.jsonHandlingService.fromJsonSafe(Currency.class, textValue); - if (currency == null) currency = this.jsonHandlingService.fromJsonSafe(Currency.class, this.cleanAsObjectString(textValue)); + Currency currency = this.tryParseJsonAsObjectString(Currency.class, textValue); if (currency == null && textValue.toUpperCase(Locale.ROOT).contains("EUR")) { currency = new Currency(); currency.setName("Euro"); @@ -433,18 +432,13 @@ public class DatasetMigrationService { } case TAGS -> { if(!this.conventionService.isNullOrEmpty(textValue)) { - Tag[] tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, textValue); - if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, this.cleanAsObjectString(textValue)); + Tag[] tags = this.tryParseJsonAsObjectString(Tag[].class, textValue); if (tags == null) { - Tag tag = this.jsonHandlingService.fromJsonSafe(Tag.class, textValue); - if (tag == null) tag = this.jsonHandlingService.fromJsonSafe(Tag.class, this.cleanAsObjectString(textValue)); + Tag tag = this.tryParseJsonAsObjectString(Tag.class, textValue); if (tag != null) tags = List.of(tag).toArray(Tag[]::new); } - if (tags == null) { - //throw new MyApplicationException("Could not parse tag : " + textValue); //TODO - logger.error("Could not parse tag : " + textValue); - break; - } + if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, this.cleanTagString(textValue)); + if (tags == null) throw new MyApplicationException("Could not parse tag : " + textValue); fieldEntity.setTextListValue(Arrays.stream(tags).toList().stream().map(Tag::getName).toList()); } } @@ -453,25 +447,193 @@ public class DatasetMigrationService { case INTERNAL_ENTRIES_DESCRIPTIONS -> throw new MyApplicationException("Found INTERNAL_ENTRIES_DMPS into description"); case REFERENCE_TYPES -> { if(!this.conventionService.isNullOrEmpty(textValue)) { //TODO - Map[] references = this.jsonHandlingService.fromJsonSafe(Map[].class, textValue); - if (references == null) references = this.jsonHandlingService.fromJsonSafe(Map[].class, this.cleanAsObjectString(textValue)); - if (references == null) { - Map reference = this.jsonHandlingService.fromJsonSafe(Map.class, textValue); - if (reference == null) reference = this.jsonHandlingService.fromJsonSafe(Map.class, this.cleanAsObjectString(textValue)); - if (reference != null) references = List.of(reference).toArray(Map[]::new); - } - if (references == null){ - logger.warn("Could not parse reference : " + textValue); - //throw new MyApplicationException("Could not parse reference : " + textValue); - break; - } + Map[] references = this.tryParseJsonAsObjectString(Map[].class, textValue); + ReferenceTypeDataEntity referenceTypeDataEntity = (ReferenceTypeDataEntity)currentField.getData(); if (referenceTypeDataEntity == null) throw new MyApplicationException("Could not parse description template reference : " + textValue); + if (references == null) { + Map reference = this.tryParseJsonAsObjectString(Map.class, textValue); + if (reference != null) references = List.of(reference).toArray(Map[]::new); + } + if (references == null && !textValue.contains("{") && !textValue.contains("[") ) { + if (!textValue.contains(",")) { + Map ref = new HashMap<>(); + ref.put("name", textValue); + + if (!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) { + ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null); + if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + ref.put("source", referenceTypeDefinition.getSources().getFirst().getKey()); + } else if (ReferenceTypeIds.DataRepositories.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "openaire"); + } else if (ReferenceTypeIds.Services.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "eosc"); + } else if (ReferenceTypeIds.Registries.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "rda-metadata-schemes"); + } else if (ReferenceTypeIds.Organizations.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "openaire"); + } else if (ReferenceTypeIds.Datasets.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "openaire"); + } else if (ReferenceTypeIds.Journal.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "openaire"); + } else if (ReferenceTypeIds.License.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "opendefinition"); + } else if (ReferenceTypeIds.PubRepositories.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "opendefinition"); + } else if (ReferenceTypeIds.Researcher.equals(referenceTypeDataEntity.getReferenceTypeId())) { + Matcher pattern = Pattern.compile("\\((.*?)\\)").matcher(textValue); + if (pattern.find()) { + String matchedString = pattern.group(0); + ref.put("name", textValue.replace(matchedString, "").trim()); + matchedString = matchedString.substring(1, matchedString.length() - 1); + ref.put("source", matchedString.split(":")[0]); + ref.put("id", matchedString.split(":")[1]); + } else { + ref.put("name", textValue); + ref.put("source", "orcid"); + } + } + if (ref.containsKey("source")) references = List.of(ref).toArray(new Map[1]); + } else { + String source = null; + if (!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) { + ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null); + if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + //if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + source = referenceTypeDefinition.getSources().getFirst().getKey(); + String[] parts = null; + if (textValue.startsWith("[\"")){ + parts = this.tryParseJsonAsObjectString(String[].class, textValue); + } else{ + if (textValue.startsWith("[")) textValue = textValue.substring(1).trim(); + if (textValue.endsWith("]")) textValue = textValue.substring(0, textValue.length() -1).trim(); + parts = textValue.split(","); + } + if (parts != null) { + List> items = new ArrayList<>(); + for (String part : parts) { + if (this.conventionService.isNullOrEmpty(part)) continue; + part = part.trim().replace(",", "").trim(); + if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue; + Map ref = new HashMap<>(); + ref.put("name", part); + ref.put("source", source); + items.add(ref); + } + if (items.isEmpty()) break; + references = items.toArray(new Map[items.size()]); + } + } + } + } + + if (references == null && !ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())){ + + if (textValue.equalsIgnoreCase("[\"dublin core\",\"{\\\"id\\\":\\\"metadataschema:rd-alliance/dublin-core\\\",\\\"label\\\":\\\"Dublin Core\\\",\\\"source\\\":\\\"\\\",\\\"uri\\\":\\\"http://dublincore.org\\\"}\"]")){ + Map ref = new HashMap<>(); + ref.put("name", "Dublin Core"); + ref.put("source", "metadataschema"); + ref.put("id", "rd-alliance/dublin-core"); + ref.put("uri", "http://dublincore.org"); + references = List.of(ref).toArray(new Map[1]); + } else if (textValue.equalsIgnoreCase("[\"Datacite\",\"{\\\"id\\\":\\\"metadataschema:rd-alliance/datacite-metadata-schema\\\",\\\"label\\\":\\\"DataCite Metadata Schema\\\",\\\"source\\\":\\\"\\\",\\\"uri\\\":\\\"http://schema.datacite.org\\\"}\"]")){ + Map ref = new HashMap<>(); + ref.put("name", "DataCite Metadata Schema"); + ref.put("source", "metadataschema"); + ref.put("id", "rd-alliance/datacite-metadata-schema"); + ref.put("uri", "http://schema.datacite.org"); + references = List.of(ref).toArray(new Map[1]); + } else if (textValue.equalsIgnoreCase("[\" .\"]")) { + break; + } else if (textValue.equalsIgnoreCase("[INT]")) { + ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null); + if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + //if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + Map ref = new HashMap<>(); + ref.put("name", "INT"); + ref.put("source", referenceTypeDefinition.getSources().getFirst().getKey()); + references = List.of(ref).toArray(new Map[1]); + } + } + + if (references == null && ReferenceTypeIds.Researcher.equals(referenceTypeDataEntity.getReferenceTypeId())){ + if (textValue.equals("[\"-\"]")){ + break; + } else if (textValue.startsWith("[\"") && !textValue.startsWith("[\"{") && textValue.contains(",")){ + references = this.tryParseJsonAsObjectString(Map[].class, textValue.replace(textValue.substring(1, textValue.indexOf(",") + 1), "")); + } else if (textValue.contains(",") && !textValue.contains("{") && !textValue.contains("[") ){ + String[] parts = textValue.split(","); + + List> items = new ArrayList<>(); + for (String part : parts){ + if (this.conventionService.isNullOrEmpty(part)) continue; + part = part.trim().replace(",", "").trim(); + if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue; + Matcher pattern = Pattern.compile("\\((.*?)\\)").matcher(part); + Map ref = new HashMap<>(); + if (pattern.find()){ + String matchedString = pattern.group(0); + ref.put("name", part.replace(matchedString, "").trim()); + matchedString = matchedString.substring(1, matchedString.length() - 1); + ref.put("source", matchedString.split(":")[0]); + ref.put("id", matchedString.split(":")[1]); + }else { + ref.put("name", part); + ref.put("source", "orcid"); + } + items.add(ref); + } + if (items.isEmpty()) break; + references = items.toArray(new Map[items.size()]); + } + } + if (references == null && (ReferenceTypeIds.Datasets.equals(referenceTypeDataEntity.getReferenceTypeId()) + || ReferenceTypeIds.DataRepositories.equals(referenceTypeDataEntity.getReferenceTypeId()) + || ReferenceTypeIds.Services.equals(referenceTypeDataEntity.getReferenceTypeId()) + || ReferenceTypeIds.License.equals(referenceTypeDataEntity.getReferenceTypeId()))){ + textValue = textValue.trim(); + if (textValue.equals("[\".\"]")){ + break; + } else if (!textValue.contains("{")) { + String[] parts = null; + if (textValue.startsWith("[\"")){ + parts = this.tryParseJsonAsObjectString(String[].class, textValue); + } else{ + if (textValue.startsWith("[")) textValue = textValue.substring(1).trim(); + if (textValue.endsWith("]")) textValue = textValue.substring(0, textValue.length() -1).trim(); + parts = textValue.split(","); + } + if (parts != null) { + List> items = new ArrayList<>(); + for (String part : parts) { + if (this.conventionService.isNullOrEmpty(part)) continue; + part = part.trim().replace(",", "").trim(); + if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue; + Map ref = new HashMap<>(); + ref.put("name", part); + if (ReferenceTypeIds.Services.equals(referenceTypeDataEntity.getReferenceTypeId())) ref.put("source", "eosc"); + else if (ReferenceTypeIds.License.equals(referenceTypeDataEntity.getReferenceTypeId())) ref.put("source", "opendefinition"); + else ref.put("source", "openaire"); + items.add(ref); + } + if (items.isEmpty()) break; + references = items.toArray(new Map[items.size()]); + } + } + } + + if (references == null){ + throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getCreated() + " " + item.getLabel() + " Could not parse reference : " + referenceTypeDataEntity.getReferenceTypeId() + " with value " + textValue); + } fieldEntity.setTextListValue(new ArrayList<>()); - for (Map item : references){ - ReferenceEntity referenceEntity = this.geReferenceEntity(item, referenceTypeDefinitionEntityMap, referenceTypeDataEntity, textValue, referenceMap); - fieldEntity.getTextListValue().add(referenceEntity.getId().toString()); + for (Map reference : references){ + ReferenceEntity referenceEntity = this.geReferenceEntity(item, reference, referenceTypeDefinitionEntityMap, referenceTypeDataEntity, referenceMap); + if (referenceEntity != null) fieldEntity.getTextListValue().add(referenceEntity.getId().toString()); } } } @@ -479,19 +641,66 @@ public class DatasetMigrationService { return fieldEntity; } - private ReferenceEntity geReferenceEntity(Map item, Map referenceTypeDefinitionEntityMap, ReferenceTypeDataEntity referenceTypeDataEntity, String textValue, Map referenceMap){ + private ReferenceEntity geReferenceEntity(Dataset dataset, Map item, Map referenceTypeDefinitionEntityMap, ReferenceTypeDataEntity referenceTypeDataEntity, Map referenceMap){ ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null); - if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + textValue); - if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + textValue); + if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(item)); + if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(item)); ReferenceEntity data = new ReferenceEntity(); if (item.containsKey("name")) data.setLabel((String) item.get("name")); if (this.conventionService.isNullOrEmpty(data.getLabel()) && item.containsKey("label")) data.setLabel((String) item.get("label")); if (item.containsKey("abbreviation")) data.setAbbreviation((String) item.get("abbreviation")); - if (item.containsKey("id")) data.setReference((String)item.get("id")); - if (item.containsKey("reference")) data.setReference((String)item.get("reference")); - if (this.conventionService.isNullOrEmpty(data.getReference()) && item.containsKey("pid")) data.setReference((String)item.get("pid")); + if (item.containsKey("pid")) data.setReference((String)item.get("pid")); + if (item.containsKey("key")) data.setSource((String)item.get("key")); + if (this.conventionService.isNullOrEmpty(data.getSource()) && item.containsKey("source")) data.setSource((String) item.get("source")); + if (item.containsKey("reference") && !this.conventionService.isNullOrEmpty(((String)item.get("reference")))){ + if (((String)item.get("reference")).contains(":")) { + String[] referenceParts = ((String)item.get("reference")).split(":", 2); + if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(referenceParts[1]); + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource(referenceParts[0]); + } + else if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(((String)item.get("reference"))); + } + if (item.containsKey("tag") && !this.conventionService.isNullOrEmpty(((String)item.get("tag")))){ + if (((String)item.get("tag")).equalsIgnoreCase("openaire")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); + } else if (((String)item.get("tag")).equalsIgnoreCase("Open Definition")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("opendefinition"); + } else if (((String)item.get("tag")).equalsIgnoreCase("RDA Metadata Schemes")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("rda-metadata-schemes"); + } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE publication repositories 1")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); + } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE publication repositories 2")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); + } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE publication repositories 3")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); + } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE Journals")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); + } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE Alternative")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); + }else if (((String)item.get("tag")).equalsIgnoreCase("Taxonomies")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("taxonomy"); + } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE Publications")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); + } else if (((String)item.get("tag")).equalsIgnoreCase("EOSC Providers")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("eosc"); + } else if (((String)item.get("tag")).equalsIgnoreCase("ORCID")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("orcid"); + } else if (((String)item.get("tag")).equalsIgnoreCase("Zenodo")) { + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("zenodo"); + } + } + if (this.conventionService.isNullOrEmpty(data.getReference()) && item.containsKey("id") && !this.conventionService.isNullOrEmpty(((String)item.get("id")))) { + if (((String)item.get("id")).startsWith("datarepo:re3data/")) { + if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(((String)item.get("id")).substring("datarepo:re3data/".length())); + if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("re3data"); + } + else if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(((String)item.get("id"))); + + data.setReference((String)item.get("id")); + } + data.setId(UUID.randomUUID()); data.setCreatedAt(Instant.now()); data.setUpdatedAt(Instant.now()); @@ -502,7 +711,7 @@ public class DatasetMigrationService { definitionEntity.setFields(new ArrayList<>()); for (Map.Entry entries : item.entrySet()){ if (entries.getValue() != null && !entries.getKey().equals("name") && !entries.getKey().equals("label")&& !entries.getKey().equals("abbreviation") - && !entries.getKey().equals("id") && !entries.getKey().equals("pid")&& !entries.getKey().equals("source")) { + && !entries.getKey().equals("id") && !entries.getKey().equals("key") && !entries.getKey().equals("pid")&& !entries.getKey().equals("source")) { eu.eudat.commons.types.reference.FieldEntity field = new eu.eudat.commons.types.reference.FieldEntity(); field.setCode(entries.getKey()); field.setDataType(ReferenceFieldDataType.Text); @@ -514,31 +723,260 @@ public class DatasetMigrationService { if (this.conventionService.isNullOrEmpty(data.getLabel())) data.setLabel(data.getReference()); - if (referenceTypeDefinition.getSources().size() == 1) data.setSource(referenceTypeDefinition.getSources().getFirst().getKey()); - else { - String key = referenceTypeDefinition.getSources().stream().filter(x-> x.getKey().equalsIgnoreCase((String) item.getOrDefault("source", null))).map(ExternalFetcherBaseSourceConfigurationEntity::getKey).findFirst().orElse(null); - if (this.conventionService.isNullOrEmpty(key)) { - logger.warn("Select first source for reference : " + textValue); - data.setSourceType(ReferenceSourceType.Internal); - data.setSource(referenceTypeDefinition.getSources().getFirst().getKey()); - //throw new MyApplicationException("Could not find reference source for reference : " + textValue); //TODO - } else { - data.setSource(key); - } + if (this.conventionService.isNullOrEmpty(data.getSource()) && + referenceTypeDefinition.getSources().size() == 1 && + !ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) { + data.setSource(referenceTypeDefinition.getSources().getFirst().getKey()); } + if (this.conventionService.isNullOrEmpty(data.getLabel()) || data.getLabel().equals("null")){ + logger.warn("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " no label for the reference : " + referenceTypeDataEntity.getReferenceTypeId() + " " + this.jsonHandlingService.toJsonSafe(item)); //TODO + return null; + } + + if (this.conventionService.isNullOrEmpty(data.getSource())) throw new MyApplicationException("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " Source not found source for reference : " + referenceTypeDataEntity.getReferenceTypeId() + " " + this.jsonHandlingService.toJsonSafe(item)); //TODO + if (this.conventionService.isNullOrEmpty(data.getReference())) { - data.setReference(this.queryFactory.query(ReferenceQuery.class).sources(data.getSource()).like(data.getLabel()).typeIds(data.getTypeId()).collectAs(new BaseFieldSet().ensure(Reference._reference, Reference._label)) + data.setReference(this.queryFactory.query(ReferenceQuery.class).sources(data.getSource()).like(data.getLabel()).typeIds(data.getTypeId()).sourceTypes(ReferenceSourceType.External).collectAs(new BaseFieldSet().ensure(Reference._reference, Reference._label)) .stream().filter(x-> x.getLabel().equals(data.getLabel())).map(ReferenceEntity::getReference).findFirst().orElse(null));//TODO } + if (this.conventionService.isNullOrEmpty(data.getReference())) { - logger.warn("Local reference created with random id" + textValue); - data.setReference(data.getId().toString().replace("-", "")); - data.setSourceType(ReferenceSourceType.Internal); //TODO + if (referenceTypeDefinition.getSources().stream().filter(x-> x.getType().equals(ReferenceTypeSourceType.API)).map(x-> (ExternalFetcherApiSourceConfigurationEntity)x).anyMatch(x-> x.getUrl().toLowerCase(Locale.ROOT).contains("eestore.paas2.uninett.no"))){ + if (data.getLabel().equalsIgnoreCase("Attribution 4.0 International")){ + data.setReference("CC-BY-4.0"); + } else if (data.getLabel().equalsIgnoreCase("Attribution-NonCommercial 4.0 International")){ + data.setReference("CC-BY-NC-4.0"); + } else if (data.getLabel().equalsIgnoreCase("Attribution-NonCommercial-NoDerivatives 4.0 International")){ + data.setReference("CC-BY-NC-ND-4.0"); + } else if (data.getLabel().equalsIgnoreCase("Attribution-NoDerivatives 4.0 International")){ + data.setReference("CC-BY-ND-4.0"); + } else if (data.getLabel().equalsIgnoreCase("Attribution-NonCommercial-ShareAlike 4.0 International") ){ + data.setReference("CC-BY-NC-SA-4.0"); + } + } if (data.getTypeId().equals(ReferenceTypeIds.License) && data.getSource().equalsIgnoreCase("OpenAIRE")){ + if (data.getLabel().equalsIgnoreCase("Academic Free License 3.0")){ + data.setReference("AFL-3.0"); + } else if (data.getLabel().equalsIgnoreCase("Attribution Assurance Licenses")){ + data.setReference("AFL-3.0"); + } + } else if (data.getTypeId().equals(ReferenceTypeIds.Datasets) && data.getSource().equalsIgnoreCase("OpenAIRE")) { + if (data.getLabel().equalsIgnoreCase("Bulk RNA-seq U2OS cells treated with small molecules") && data.getSource().equalsIgnoreCase("OpenAIRE")){ + data.setReference("50|_____OmicsDI::7fc1bcd133082463ec1948b82f1ad0ee"); + } else if (data.getLabel().equalsIgnoreCase("Genome-wide effect of the microenvironment over the Galectin-7 regulation network in HeLa and SiHa tumors in nude mice.")){ + data.setReference("50|_____OmicsDI::4e0de948bf961b96dbdef82797b5a8dc"); + } +// else if (data.getLabel().equalsIgnoreCase("other")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("none")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("to be clarified later")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("articles")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Freistilringen")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } + else if (data.getLabel().equalsIgnoreCase("Randomized Phase 3 Trial of ALIMTA (Pemetrexed) and Carboplatin versus Etoposide and Carboplatin in Extensive-Stage Small Cell Lung Cancer")){ + data.setReference("50|datacite____::1df6f72ab95712d5f26ecb43b2d1d303"); + } + } else if (data.getTypeId().equals(ReferenceTypeIds.DataRepositories) && data.getSource().equalsIgnoreCase("OpenAIRE")) { + if (data.getLabel().equalsIgnoreCase("INPTDAT")){ + data.setReference("re3data_____::r3d100013120"); + } if (data.getLabel().equalsIgnoreCase("OpenAPC Global Initiative")){ + data.setReference("apc_________::openapc"); + } else if (data.getLabel().equalsIgnoreCase("Human Gene and Protein Database")){ + data.setReference("fairsharing_::1592"); + } else if (data.getLabel().equalsIgnoreCase("Bacterial protein tYrosine Kinase database")){ + data.setReference("fairsharing_::1557"); + } +// else if (data.getLabel().equalsIgnoreCase("other")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Zenodo")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Zenado")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("none")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("articles")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("https://github.com/sisyphe-re/riot_rpl_udp_scenarios")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else else if (data.getLabel().equalsIgnoreCase("Cool data")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Raw data")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Repository hosted by Zenodo")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("ICPSR - Interuniversity Consortium for Political and Social Research")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("XC7 pharmaceutical development IPF")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("nct")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("UK Data Service")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("figshare")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("University of Strathclyde")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } + } else if (data.getTypeId().equals(ReferenceTypeIds.Services) && data.getSource().equalsIgnoreCase("eosc")) { +// if (data.getLabel().equalsIgnoreCase("website")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("integration1")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("SeaDataNet Common Data Index (CDI)")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Europeana APIs")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Imaging at Plentzia Marine Station-UPV/EHU")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Spectroscopy")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Library services at Stazione Zoologica Anton Dohrn")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("PRACE Training Portal")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("DARIAH Science Gateway")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Finding Anisotropy Tensor")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Imaging at University of Bergen-SLRC")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Training facilities at Toralla Marine Science Station - Vigo University Marine Research Centre")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Zenodo")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Digital Commons")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } + }else if (data.getTypeId().equals(ReferenceTypeIds.Registries) && data.getSource().equalsIgnoreCase("rda-metadata-schemes")) { +// if (data.getLabel().equalsIgnoreCase("DataCite Metadata Schema")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("AgMES (Agricultural Metadata Element Set)")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } + } else if (data.getTypeId().equals(ReferenceTypeIds.Organizations) && data.getSource().equalsIgnoreCase("OpenAIRE")) { + if (data.getLabel().equalsIgnoreCase("Programa de Pós-Graduação em Ciência da Informação Ibict/UFRJ")){ + data.setReference("pending_org_::f20d38c291e7d4a7a11d08481eebac44"); + } else if (data.getLabel().equalsIgnoreCase("Applied Decision Science (United States)")){ + data.setReference("openorgs____::0000093299"); + } else if (data.getLabel().equalsIgnoreCase("BIODOL THERAPEUTICS")){ + data.setReference("pending_org_::4c550b2917a6fc395bdb79eccdafe950"); + } else if (data.getLabel().equalsIgnoreCase("OKKO")){ + data.setReference("pending_org_::37ddf98c5fd2d6373fd315b0371fd4ac"); + } else if (data.getLabel().equalsIgnoreCase("Health Research Board")){ + data.setReference("openorgs____::0000006166"); + } else if (data.getLabel().equalsIgnoreCase("Vidzeme University of Applied Sciences")){ + data.setReference("openorgs____::0000029719"); + } +// else if (data.getLabel().equalsIgnoreCase("Other Lab")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("ATS APPLIED TECH SYSTEMS LTD")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } else if (data.getLabel().equalsIgnoreCase("Universidade Nova de Lisboa Centro de Linguística")){ +// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); +// data.setSourceType(ReferenceSourceType.Internal); +// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); +// } + } else if (data.getTypeId().equals(ReferenceTypeIds.Researcher)){ + if (data.getLabel().equalsIgnoreCase("Radosław Cichocki")){ + data.setReference("0000-0002-4667-8445"); + } else if (data.getLabel().equalsIgnoreCase("Kiranmai Uppuluri")){ + data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); + data.setSourceType(ReferenceSourceType.Internal); + if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); + } + } + + } + + if (this.conventionService.isNullOrEmpty(data.getReference())) { + data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); + data.setSourceType(ReferenceSourceType.Internal); + if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); + logger.warn("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " reference auto created as internal " + referenceTypeDataEntity.getReferenceTypeId() + " for value " + this.jsonHandlingService.toJsonSafe(item)); } ReferenceEntity existingEntity = referenceMap.getOrDefault(new ReferenceKey(data), null); if(existingEntity == null){ - referenceMap.put(new ReferenceKey(data), data); + if (data.getCreatedById() != null) referenceMap.put(new ReferenceKey(data), data); this.entityManager.persist(data); return data; } else { @@ -547,15 +985,79 @@ public class DatasetMigrationService { } - private String cleanAsObjectString(String value){ - value = value.trim().replace("\\", ""); - value = value.trim().replace("[\"{", "[{"); - value = value.trim().replace("}\"]", "}]"); - value = value.trim().replace("}\", \"{", "}, {"); - value = value.trim().replace("}\",\"{", "},{"); + private T tryParseJsonAsObjectString(Class type, String value){ + T item = this.jsonHandlingService.fromJsonSafe(type, value); + if (item == null) item = this.jsonHandlingService.fromJsonSafe(type, StringEscapeUtils.unescapeJava(value)); + if (item == null) item = this.jsonHandlingService.fromJsonSafe(type, StringEscapeUtils.unescapeJson(value)); + if (item == null) { + String newValue = StringEscapeUtils.unescapeJava(value); + newValue = newValue.trim().replace("[\"{", "[{"); + newValue = newValue.trim().replace("}\"]", "}]"); + newValue = newValue.trim().replace("}\", \"{", "}, {"); + newValue = newValue.trim().replace("}\",\"{", "},{"); + newValue = newValue.trim().replace(", \"{", ", {"); + newValue = newValue.trim().replace(",\"{", ",{"); + newValue = newValue.trim().replace("}\",", "},"); + + while (!this.conventionService.isNullOrEmpty(newValue) && newValue.startsWith("\"")) newValue = newValue.substring(1); + while (!this.conventionService.isNullOrEmpty(newValue) && newValue.endsWith("\"")) newValue = newValue.substring(0, newValue.length() - 1); + item = this.jsonHandlingService.fromJsonSafe(type, newValue); + } + if (item == null) { + String newValue = StringEscapeUtils.unescapeJson(value); + newValue = newValue.trim().replace("[\"{", "[{"); + newValue = newValue.trim().replace("}\"]", "}]"); + newValue = newValue.trim().replace("}\", \"{", "}, {"); + newValue = newValue.trim().replace("}\",\"{", "},{"); + newValue = newValue.trim().replace(", \"{", ", {"); + newValue = newValue.trim().replace(",\"{", ",{"); + newValue = newValue.trim().replace("}\",", "},"); + + while (!this.conventionService.isNullOrEmpty(newValue) && newValue.startsWith("\"")) newValue = newValue.substring(1); + while (!this.conventionService.isNullOrEmpty(newValue) && newValue.endsWith("\"")) newValue = newValue.substring(0, newValue.length() - 1); + item = this.jsonHandlingService.fromJsonSafe(type, newValue); + } + if (item == null) { + String newValue = value.trim().replace("\\", ""); + newValue = newValue.trim().replace("[\"{", "[{"); + newValue = newValue.trim().replace("}\"]", "}]"); + newValue = newValue.trim().replace("}\", \"{", "}, {"); + newValue = newValue.trim().replace("}\",\"{", "},{"); + newValue = newValue.trim().replace(", \"{", ", {"); + newValue = newValue.trim().replace(",\"{", ",{"); + newValue = newValue.trim().replace("}\",", "},"); + + while (!this.conventionService.isNullOrEmpty(newValue) && newValue.startsWith("\"")) newValue = newValue.substring(1); + while (!this.conventionService.isNullOrEmpty(newValue) && newValue.endsWith("\"")) newValue = newValue.substring(0, newValue.length() - 1); + item = this.jsonHandlingService.fromJsonSafe(type, newValue); + } + if (item == null) { + String newValue = value.trim().replace("=", ":"); + newValue = newValue.trim().replace("{", "{\""); + newValue = newValue.trim().replace("}", "\"}"); + newValue = newValue.trim().replace(":", "\":\""); + newValue = newValue.trim().replace(", ", "\", \""); + newValue = newValue.trim().replace("}\", \"{", "}, {"); + newValue = newValue.trim().replace("\"\"", "\""); + newValue = newValue.trim().replace(":\"null,", ":null,"); + newValue = newValue.trim().replace(":\"null\",", ":null,"); + newValue = newValue.trim().replace(":\"null\"", ":null"); + newValue = newValue.trim().replace(":\"null}\"", ":null}"); + newValue = newValue.trim().replace("https\":\"", "https:"); + newValue = newValue.trim().replace("datarepo\":\"", "datarepo:"); + + item = this.jsonHandlingService.fromJsonSafe(type, newValue); + } + return item; + } - while (!this.conventionService.isNullOrEmpty(value) && value.startsWith("\"")) value = value.substring(1); - while (!this.conventionService.isNullOrEmpty(value) && value.endsWith("\"")) value = value.substring(0, value.length() - 1); + private String cleanTagString(String value){ + value = value.trim().replace("=", ":"); + value = value.trim().replace("{", "{\""); + value = value.trim().replace("}", "\"}"); + value = value.trim().replace(":", "\":\""); + value = value.trim().replace(", ", "\", \""); + value = value.trim().replace("}\", \"{", "}, {"); return value; } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java index 86311e908..806b1c618 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java @@ -82,14 +82,14 @@ public class DescriptionTemplateXmlMigrationService { Document document = XmlBuilder.fromXml(item.getDefinition()); if (document == null){ logger.error("Migrate DescriptionTemplate " + item.getId() + " failed read xml"); - continue; + throw new MyApplicationException("Migrate DescriptionTemplate " + item.getId() + " failed read xml"); } ViewStyleModel viewStyleModel = null; try { viewStyleModel = new ViewStyleModel().fromXml(document.getDocumentElement()); } catch (Exception ex){ logger.error("Migrate DescriptionTemplate " + item.getId() + " failed read xml"); - continue; + throw new MyApplicationException("Migrate DescriptionTemplate " + item.getId() + " failed read xml"); } item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(viewStyleModel, referenceTypeEntityMap))); this.entityManager.merge(item); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java index c7a5885b6..6d2dd5e7f 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java @@ -78,23 +78,19 @@ public class DmpDatasetProfileMigrationService { DmpDatasetProfileData profileData = jsonHandlingService.fromJson(DmpDatasetProfileData.class, item.getData()); if (profileData == null || profileData.dmpSectionIndex == null || profileData.dmpSectionIndex.isEmpty()){ - logger.error("Migrate DmpDatasetProfile " + item.getId() + " failed no section info "); throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " failed no section info "); } DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null); if (definition == null){ - logger.error("Migrate DmpDatasetProfile " + item.getId() + " failed blueprint definition not found for blueprint " + item.getDmp().getProfile().getId()); throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " failed blueprint definition not found for blueprint " + item.getDmp().getProfile().getId()); } for (int sectionIndex: profileData.dmpSectionIndex) { if (definition.getSections() == null || definition.getSections().size() <= sectionIndex) { - logger.error("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex); throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex); } UUID sectionId = definition.getSections().get(sectionIndex).getId(); if (sectionId == null) { - logger.error("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex); throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex); } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java index 2ab756b4f..2f21e9872 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java @@ -172,7 +172,6 @@ public class DmpMigrationService { data.setLanguage("en"); } if (data.getCreatorId() == null){ - logger.warn("Migration skipped creator not found " + item.getId()); throw new MyApplicationException("Migration skipped creator not found " + item.getId()); } @@ -196,12 +195,10 @@ public class DmpMigrationService { if (organisations == null || organisations.isEmpty()) return; if (referenceTypeFieldEntities == null) { - logger.warn("Migration failed Blueprint not found " + item.getProfile().getId()); throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId()); } ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Organizations)).findFirst().orElse(null); if (referenceTypeFieldEntity == null) { - logger.warn("Migration failed Organizations field not found " + item.getId()); throw new MyApplicationException("Migration failed Organizations field not found " + item.getId()); } @@ -222,12 +219,10 @@ public class DmpMigrationService { Project project = item.getProject(); if (project == null) return; if (referenceTypeFieldEntities == null) { - logger.warn("Migration failed Blueprint not found " + item.getProfile().getId()); throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId()); } ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Project)).findFirst().orElse(null); if (referenceTypeFieldEntity == null) { - logger.warn("Migration failed Project field not found " + item.getId()); throw new MyApplicationException("Migration failed Project field not found " + item.getId()); } @@ -246,12 +241,10 @@ public class DmpMigrationService { Set researchers = item.getResearchers(); if (researchers == null || researchers.isEmpty()) return; if (referenceTypeFieldEntities == null) { - logger.warn("Migration failed Blueprint not found " + item.getProfile().getId()); throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId()); } ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Researcher)).findFirst().orElse(null); if (referenceTypeFieldEntity == null) { - logger.warn("Migration failed Researchers field not found " + item.getId()); throw new MyApplicationException("Migration failed Researchers field not found " + item.getId()); } @@ -273,12 +266,10 @@ public class DmpMigrationService { if (grant == null) return; if (referenceTypeFieldEntities == null) { - logger.warn("Migration failed Blueprint not found " + item.getProfile().getId()); throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId()); } ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Grants)).findFirst().orElse(null); if (referenceTypeFieldEntity == null) { - logger.warn("Migration failed Grant field not found " + item.getId()); throw new MyApplicationException("Migration failed Grant field not found " + item.getId()); } @@ -294,7 +285,6 @@ public class DmpMigrationService { if (grant.getFunder() != null){ ReferenceTypeFieldEntity funderField = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Funder)).findFirst().orElse(null); if (funderField == null) { - logger.warn("Migration failed Funder field not found " + item.getId()); throw new MyApplicationException("Migration failed Funder field not found " + item.getId()); } DmpReferenceEntity founder = new DmpReferenceEntity(); @@ -313,12 +303,10 @@ public class DmpMigrationService { if (model.getExtraProperties() != null) { if (model.getExtraProperties().containsKey("license") && model.getExtraProperties().get("license") != null) { if (referenceTypeFieldEntities == null) { - logger.warn("Migration failed Blueprint not found " + item.getProfile().getId()); throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId()); } ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.License)).findFirst().orElse(null); if (referenceTypeFieldEntity == null) { - logger.warn("Migration failed License field not found " + item.getId()); throw new MyApplicationException("Migration failed License field not found " + item.getId()); } @@ -388,7 +376,6 @@ public class DmpMigrationService { referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity)); licenseIdByName.put(licensePid, referenceEntity.getId()); - logger.debug("License '{}' migrated", licensePid); } DmpReferenceEntity dmpReferenceEntity = new DmpReferenceEntity(); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ExternalDatasetMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ExternalDatasetMigrationService.java index a864fe3be..c2ee98c9f 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ExternalDatasetMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ExternalDatasetMigrationService.java @@ -42,7 +42,7 @@ public class ExternalDatasetMigrationService { entityManager.detach(item); if (item.getReference() == null || !item.getReference().contains(":")){ logger.warn("Reference generated because is null ExternalDataset " + item.getId()); - item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); } eu.old.eudat.models.data.externaldataset.ExternalDatasetModel model = new eu.old.eudat.models.data.externaldataset.ExternalDatasetModel().fromDataModel(item); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/FunderMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/FunderMigrationService.java index c56e89e6f..0f5b55ccc 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/FunderMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/FunderMigrationService.java @@ -46,7 +46,7 @@ public class FunderMigrationService { entityManager.detach(item); if (item.getReference() == null || !item.getReference().contains(":")){ logger.warn("Reference generated because is null Funder " + item.getId()); - item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); } eu.old.eudat.models.data.funder.Funder model = new eu.old.eudat.models.data.funder.Funder().fromDataModel(item); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/GrantMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/GrantMigrationService.java index 1a52f578b..704339566 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/GrantMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/GrantMigrationService.java @@ -52,7 +52,7 @@ public class GrantMigrationService { entityManager.detach(item); if (item.getReference() == null || !item.getReference().contains(":")){ logger.warn("Reference generated because is null Grant " + item.getId()); - item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); } String[] referenceParts = item.getReference().split(":", 2); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/OrganizationMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/OrganizationMigrationService.java index 5c1253e58..f7da6ad71 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/OrganizationMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/OrganizationMigrationService.java @@ -51,7 +51,7 @@ public class OrganizationMigrationService { entityManager.detach(item); if (item.getReference() == null || !item.getReference().contains(":")){ logger.warn("Reference generated because is null Organisation " + item.getId()); - item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); } eu.old.eudat.models.data.dmp.Organisation model = new eu.old.eudat.models.data.dmp.Organisation().fromDataModel(item); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ProjectMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ProjectMigrationService.java index 3f091dbb2..10b1e2c71 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ProjectMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ProjectMigrationService.java @@ -52,7 +52,7 @@ public class ProjectMigrationService { entityManager.detach(item); if (item.getReference() == null || !item.getReference().contains(":")){ logger.warn("Reference generated because is null Project " + item.getId()); - item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); } eu.old.eudat.models.data.project.Project model = new eu.old.eudat.models.data.project.Project().fromDataModel(item); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ReferenceTypeIds.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ReferenceTypeIds.java index 17fa3e1db..43758cd12 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ReferenceTypeIds.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ReferenceTypeIds.java @@ -1,5 +1,6 @@ package eu.old.eudat.migration; +import java.util.List; import java.util.UUID; public class ReferenceTypeIds { @@ -18,4 +19,7 @@ public class ReferenceTypeIds { public static UUID Journal = UUID.fromString("8ec7556b-749d-4c4a-a4b9-43d064693795"); public static UUID PubRepositories = UUID.fromString("1e927daa-b856-443f-96da-22f325f7322f"); + public static List KnownReferenceTypeIds = List.of(DataRepositories, Datasets, Funder, Grants, Organizations, + Project, Registries, Researcher, Services, License, Taxonomy, Publication, Journal, PubRepositories); + } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/RegistryMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/RegistryMigrationService.java index 485e54b9b..90367c08e 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/RegistryMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/RegistryMigrationService.java @@ -51,7 +51,7 @@ public class RegistryMigrationService { entityManager.detach(item); if (item.getReference() == null || !item.getReference().contains(":")){ logger.warn("Reference generated because is null Registry " + item.getId()); - item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); } eu.old.eudat.models.data.registries.RegistryModel model = new eu.old.eudat.models.data.registries.RegistryModel().fromDataModel(item); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ResearcherMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ResearcherMigrationService.java index 48ae4060e..497aef351 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ResearcherMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ResearcherMigrationService.java @@ -52,7 +52,7 @@ public class ResearcherMigrationService { entityManager.detach(item); if (item.getReference() == null || !item.getReference().contains(":")){ logger.warn("Reference generated because is null Researcher " + item.getId()); - item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); } eu.old.eudat.models.data.dmp.Researcher model = new eu.old.eudat.models.data.dmp.Researcher().fromDataModel(item); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ServiceMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ServiceMigrationService.java index d800b9255..28e686ecd 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ServiceMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ServiceMigrationService.java @@ -50,7 +50,7 @@ public class ServiceMigrationService { entityManager.detach(item); if (item.getReference() == null || !item.getReference().contains(":")){ logger.warn("Reference generated because is null Service " + item.getId()); - item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); } eu.old.eudat.models.data.services.ServiceModel model = new eu.old.eudat.models.data.services.ServiceModel().fromDataModel(item); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java index 2b411afb2..5d2bb1828 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java @@ -51,17 +51,15 @@ public class TagMigrationService { Set savedTagNames = new HashSet<>(); Map savedTagIdsByName = new HashMap<>(); - DatasetCriteria criteria = new DatasetCriteria(); List items; do { items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); if (items != null && !items.isEmpty()) { logger.debug("Migrate Dataset tags " + page * PageSize + " of " + total); - criteria.setIds(items.stream().map(Dataset::getId).toList()); List elasticDatasets = this.datasetRepository.findByIds(items.stream().map(x -> x.getId().toString()).toList()); - for (Dataset item : items) { + for (Dataset item : items) { List found = elasticDatasets.stream().filter(x -> item.getId().toString().equals(x.getId())).toList(); if (found.isEmpty()) { logger.error("No dataset with id {} found on elastic search. Skipping tag migration for this dataset", item.getId()); @@ -70,7 +68,7 @@ public class TagMigrationService { eu.old.eudat.elastic.entities.Dataset elasticDataset = found.getFirst(); boolean tagAlreadyExists; if (elasticDataset.getTags() != null && !elasticDataset.getTags().isEmpty()) { - for(Tag tag : elasticDataset.getTags()) { + for (Tag tag : elasticDataset.getTags()) { tagAlreadyExists = savedTagNames.contains(tag.getName()); //TODO maybe should check owner to recreate savedTagNames.add(tag.getName()); if (!tagAlreadyExists) { @@ -83,7 +81,6 @@ public class TagMigrationService { tagEntity.setIsActive(IsActive.Active); savedTagIdsByName.put(tag.getName(), tagEntity.getId()); this.entityManager.persist(tagEntity); - logger.debug("New tag saved '{}'", tag.getName()); } DescriptionTagEntity descriptionTagEntity = new DescriptionTagEntity(); descriptionTagEntity.setId(UUID.randomUUID()); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java index a27c4ddba..a99917fd8 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java @@ -90,6 +90,7 @@ public class MigrationController { @GetMapping("all") @Transactional public boolean migrateAll() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException, NoSuchFieldException, InvalidApplicationException, TransformerException, URISyntaxException { +// //Reference // this.dataRepositoryMigrationService.migrate(); // this.externalDatasetMigrationService.migrate(); // this.funderMigrationService.migrate(); @@ -100,6 +101,7 @@ public class MigrationController { // this.researcherMigrationService.migrate(); // this.serviceMigrationService.migrate(); // +// //User // this.userContactInfoMigrationService.migrate(); // this.userMigrationService.migrate(); // @@ -107,188 +109,190 @@ public class MigrationController { // this.dmpBlueprintXmlMigrationService.migrate(); // this.descriptionTemplateXmlMigrationService.migrate(); // +// //Dmp // this.dmpMigrationService.migrate(); // this.dmpDatasetProfileMigrationService.migrate(); - this.datasetMigrationService.migrate(); //TODO -// this.tagMigrationService.migrate(); -// // this.dmpUserMigrationService.migrate(); -// + + //Description + this.datasetMigrationService.migrate(); //TODO // this.referenceMigrationService.migrateDatasetReferences(); -// +// this.tagMigrationService.migrate(); + // this.storageFileMigrationService.migrate(); - return true; + throw new RuntimeException(""); + //return true; } - @GetMapping("dmp-blueprints-xml") - @Transactional - public boolean migrateDmpBlueprint() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException { - this.dmpBlueprintXmlMigrationService.migrate(); - return true; - - } - - @GetMapping("storage-files") - @Transactional - public boolean migrateStorageFile() throws IOException { - this.storageFileMigrationService.migrate(); - return true; - - } - - @GetMapping("description-template-xml") - @Transactional - public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { - this.descriptionTemplateXmlMigrationService.migrate(); - return true; - - } - - @GetMapping("description-template-xml-clean-invalid-reference-types") - @Transactional - public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { - this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate(); - return true; - - } - - @GetMapping("contacts") - @Transactional - public boolean migrateContacts() { - this.userContactInfoMigrationService.migrate(); - return true; - - } - - @GetMapping("users") - @Transactional - public boolean migrateUsers() throws JsonProcessingException { - this.userMigrationService.migrate(); - return true; - - } - - - @GetMapping("dmps") - @Transactional - public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException { - this.dmpMigrationService.migrate(); - return true; - } - - @GetMapping("datasets") - @Transactional - public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { - this.datasetMigrationService.migrate(); - return true; - } - - @GetMapping("tags") - @Transactional - public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { - this.tagMigrationService.migrate(); - return true; - } - - @GetMapping("dmp-dataset-profiles") - @Transactional - public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { - this.dmpDatasetProfileMigrationService.migrate(); - return true; - } - - @GetMapping("references") - @Transactional - public boolean migrateReferences() { - this.dataRepositoryMigrationService.migrate(); - this.externalDatasetMigrationService.migrate(); - this.funderMigrationService.migrate(); - this.grantMigrationService.migrate(); - this.organizationMigrationService.migrate(); - this.projectMigrationService.migrate(); - this.registryMigrationService.migrate(); - this.researcherMigrationService.migrate(); - this.serviceMigrationService.migrate(); - return true; - } - - @GetMapping("dmp-users") - @Transactional - public boolean migrateDmpUsers() { - this.dmpUserMigrationService.migrate(); - return true; - } - - @GetMapping("dataset-references") - @Transactional - public boolean migrateDatasetReferences() { - this.referenceMigrationService.migrateDatasetReferences(); - return true; - } - - @GetMapping("organizations") - @Transactional - public boolean migrateOrganizations() { - this.organizationMigrationService.migrate(); - return true; - } - - - @GetMapping("dataRepositories") - @Transactional - public boolean migrateDataRepositories() { - this.dataRepositoryMigrationService.migrate(); - return true; - } - - @GetMapping("externalDatasets") - @Transactional - public boolean migrateExternalDatasets() { - this.externalDatasetMigrationService.migrate(); - return true; - } - - @GetMapping("funders") - @Transactional - public boolean migrateFunders() { - this.funderMigrationService.migrate(); - return true; - } - - @GetMapping("grants") - @Transactional - public boolean migrateGrants() { - this.grantMigrationService.migrate(); - return true; - } - - @GetMapping("projects") - @Transactional - public boolean migrateProjects() { - this.projectMigrationService.migrate(); - return true; - } - - @GetMapping("registries") - @Transactional - public boolean migrateRegistries() { - this.registryMigrationService.migrate(); - return true; - } - - @GetMapping("researchers") - @Transactional - public boolean migrateResearchers() { - this.researcherMigrationService.migrate(); - return true; - } - - @GetMapping("services") - @Transactional - public boolean migrateServices() { - this.serviceMigrationService.migrate(); - return true; - } +// @GetMapping("dmp-blueprints-xml") +// @Transactional +// public boolean migrateDmpBlueprint() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException { +// this.dmpBlueprintXmlMigrationService.migrate(); +// return true; +// +// } +// +// @GetMapping("storage-files") +// @Transactional +// public boolean migrateStorageFile() throws IOException { +// this.storageFileMigrationService.migrate(); +// return true; +// +// } +// +// @GetMapping("description-template-xml") +// @Transactional +// public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { +// this.descriptionTemplateXmlMigrationService.migrate(); +// return true; +// +// } +// +// @GetMapping("description-template-xml-clean-invalid-reference-types") +// @Transactional +// public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { +// this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate(); +// return true; +// +// } +// +// @GetMapping("contacts") +// @Transactional +// public boolean migrateContacts() { +// this.userContactInfoMigrationService.migrate(); +// return true; +// +// } +// +// @GetMapping("users") +// @Transactional +// public boolean migrateUsers() throws JsonProcessingException { +// this.userMigrationService.migrate(); +// return true; +// +// } +// +// +// @GetMapping("dmps") +// @Transactional +// public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException { +// this.dmpMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("datasets") +// @Transactional +// public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { +// this.datasetMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("tags") +// @Transactional +// public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { +// this.tagMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("dmp-dataset-profiles") +// @Transactional +// public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { +// this.dmpDatasetProfileMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("references") +// @Transactional +// public boolean migrateReferences() { +// this.dataRepositoryMigrationService.migrate(); +// this.externalDatasetMigrationService.migrate(); +// this.funderMigrationService.migrate(); +// this.grantMigrationService.migrate(); +// this.organizationMigrationService.migrate(); +// this.projectMigrationService.migrate(); +// this.registryMigrationService.migrate(); +// this.researcherMigrationService.migrate(); +// this.serviceMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("dmp-users") +// @Transactional +// public boolean migrateDmpUsers() { +// this.dmpUserMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("dataset-references") +// @Transactional +// public boolean migrateDatasetReferences() { +// this.referenceMigrationService.migrateDatasetReferences(); +// return true; +// } +// +// @GetMapping("organizations") +// @Transactional +// public boolean migrateOrganizations() { +// this.organizationMigrationService.migrate(); +// return true; +// } +// +// +// @GetMapping("dataRepositories") +// @Transactional +// public boolean migrateDataRepositories() { +// this.dataRepositoryMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("externalDatasets") +// @Transactional +// public boolean migrateExternalDatasets() { +// this.externalDatasetMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("funders") +// @Transactional +// public boolean migrateFunders() { +// this.funderMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("grants") +// @Transactional +// public boolean migrateGrants() { +// this.grantMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("projects") +// @Transactional +// public boolean migrateProjects() { +// this.projectMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("registries") +// @Transactional +// public boolean migrateRegistries() { +// this.registryMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("researchers") +// @Transactional +// public boolean migrateResearchers() { +// this.researcherMigrationService.migrate(); +// return true; +// } +// +// @GetMapping("services") +// @Transactional +// public boolean migrateServices() { +// this.serviceMigrationService.migrate(); +// return true; +// } }