package eu.old.eudat.migration; import eu.eudat.commons.JsonHandlingService; import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.enums.*; import eu.eudat.commons.types.description.*; import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity; import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity; import eu.eudat.commons.types.dmpblueprint.DefinitionEntity; import eu.eudat.commons.types.externalfetcher.ExternalFetcherApiSourceConfigurationEntity; import eu.eudat.commons.types.referencetype.ReferenceTypeDefinitionEntity; import eu.eudat.convention.ConventionService; import eu.eudat.data.*; import eu.eudat.model.Dmp; import eu.eudat.model.Reference; import eu.eudat.model.ReferenceType; import eu.eudat.query.*; import eu.old.eudat.data.dao.entities.DatasetDao; import eu.old.eudat.data.entities.Dataset; import eu.old.eudat.logic.services.operations.DatabaseRepository; import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.fieldset.BaseFieldSet; import gr.cite.tools.logging.LoggerService; import jakarta.persistence.EntityManager; import jakarta.xml.bind.JAXBException; import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @Service public class DatasetMigrationService { private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class)); private final DatabaseRepository databaseRepository; private final JsonHandlingService jsonHandlingService; private final QueryFactory queryFactory; private final XmlHandlingService xmlHandlingService; private static final int PageSize = 500; private static final boolean TestMode = false; private final EntityManager entityManager; private final ConventionService conventionService; private final MigrationTools migrationTools; public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager, ConventionService conventionService, MigrationTools migrationTools) { this.databaseRepository = databaseRepository; this.jsonHandlingService = jsonHandlingService; this.queryFactory = queryFactory; this.xmlHandlingService = xmlHandlingService; this.entityManager = entityManager; this.conventionService = conventionService; this.migrationTools = migrationTools; } public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { DatasetDao datasetDao = databaseRepository.getDatasetDao(); long total = datasetDao.asQueryable().count(); logger.debug("Migrate Dataset Total : " + total); int page = 0; Map referenceTypeDefinitionEntityMap = new HashMap<>(); List referenceTypes = this.queryFactory.query(ReferenceTypeQuery.class).collectAs(new BaseFieldSet().ensure(ReferenceType._id).ensure(ReferenceType._definition)); for (ReferenceTypeEntity referenceType : referenceTypes){ referenceTypeDefinitionEntityMap.put(referenceType.getId(), this.xmlHandlingService.fromXml(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition())); } Map referenceMap = new HashMap<>(); List references = this.queryFactory.query(ReferenceQuery.class).collectAs(new BaseFieldSet().ensure(Reference._id).ensure(Reference._source).ensure(Reference._type).ensure(Reference._reference)); for (ReferenceEntity reference : references){ if (reference.getCreatedById() == null) referenceMap.put(new ReferenceKey(reference), reference); } List items; do { items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); if (items != null && !items.isEmpty()) { logger.debug("Migrate Dataset " + page * PageSize + " of " + total); List dmpBlueprints = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().map(x-> x.getDmp().getProfile().getId()).distinct().toList()).collect(); Map dmpBlueprintsMap = new HashMap<>(); for (DmpBlueprintEntity dmpBlueprint : dmpBlueprints) { DefinitionEntity definitionEntity = this.xmlHandlingService.fromXml(DefinitionEntity.class, dmpBlueprint.getDefinition()); dmpBlueprintsMap.put(dmpBlueprint.getId(), definitionEntity); } List dmpDescriptionTemplateEntities = this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collect(); List descriptionTemplates = this.queryFactory.query(DescriptionTemplateQuery.class).ids(items.stream().map(x-> x.getProfile().getId()).distinct().toList()).collect(); Map creatorsByDmp = this.queryFactory.query(DmpQuery.class).ids(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collectAs(new BaseFieldSet().ensure(Dmp._id).ensure(Dmp._creator)).stream().collect(Collectors.toMap(DmpEntity::getId, DmpEntity::getCreatorId)); Map descriptionTemplateDefinitionMap = new HashMap<>(); for (DescriptionTemplateEntity descriptionTemplateEntity : descriptionTemplates){ descriptionTemplateDefinitionMap.put(descriptionTemplateEntity.getId(), this.xmlHandlingService.fromXml(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition())); } for (Dataset item : items) { DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null); if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) { throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); } UUID sectionId = definition.getSections().get(item.getDmpSectionIndex()).getId(); if (sectionId == null) { throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); } List itemDescriptionTemplates = this.getOrCreateDmpDescriptionTemplateEntity(item, sectionId, dmpDescriptionTemplateEntities); if (itemDescriptionTemplates.size() > 1) { throw new MyApplicationException("Migrate Dataset " + item.getId() + " multiple DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); } DescriptionEntity data = new DescriptionEntity(); data.setId(item.getId()); data.setDescription(item.getDescription()); if (item.getCreator() != null) { data.setCreatedById(item.getCreator().getId()); } else { data.setCreatedById(creatorsByDmp.getOrDefault(item.getDmp().getId(), null)); } data.setDmpId(item.getDmp().getId()); data.setLabel(item.getLabel()); data.setDmpDescriptionTemplateId(itemDescriptionTemplates.getFirst().getId()); data.setDescriptionTemplateId(item.getProfile().getId()); data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now()); data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now()); if (item.getFinalizedAt() != null) data.setFinalizedAt(item.getFinalizedAt().toInstant()); if (item.getStatus() == 99) { data.setIsActive(IsActive.Inactive); data.setStatus(DescriptionStatus.Canceled); } else { data.setIsActive(IsActive.Active); data.setStatus(DescriptionStatus.of(item.getStatus())); } eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity = descriptionTemplateDefinitionMap.getOrDefault(item.getProfile().getId(), null); data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap))); if (data.getCreatedById() == null){ throw new MyApplicationException("Migration skipped creator not found " + item.getId()); } this.entityManager.persist(data); this.entityManager.flush(); } page++; } } while (items != null && !items.isEmpty() && !TestMode); } private List getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List dmpDescriptionTemplateEntities){ List itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList(); if (itemDescriptionTemplates.isEmpty()) { logger.warn("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); if (dmpDescriptionTemplateEntities.stream().anyMatch(x -> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId))) { DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity(); dmpDescriptionTemplateEntity.setId(UUID.randomUUID()); dmpDescriptionTemplateEntity.setDescriptionTemplateGroupId(item.getProfile().getGroupId()); dmpDescriptionTemplateEntity.setDmpId(item.getDmp().getId()); dmpDescriptionTemplateEntity.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now()); dmpDescriptionTemplateEntity.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now()); dmpDescriptionTemplateEntity.setSectionId(sectionId); dmpDescriptionTemplateEntity.setIsActive(IsActive.Active); this.entityManager.persist(dmpDescriptionTemplateEntity); itemDescriptionTemplates = List.of(dmpDescriptionTemplateEntity); } else { throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getLabel() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); } } return itemDescriptionTemplates; } private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, Map referenceTypeDefinitionEntityMap, Map referenceMap) { if (this.conventionService.isNullOrEmpty(item.getProperties())) return null; JSONObject jObject = new JSONObject(item.getProperties()); Map properties = jObject.toMap(); PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity(); propertyDefinitionEntity.setFieldSets(new HashMap<>()); List fieldSetEntities = descriptionTemplateDefinitionEntity.getAllFieldSets(); for (String key : properties.keySet()) { if (key.toLowerCase(Locale.ROOT).trim().startsWith("commentfieldvalue")){ if (properties.get(key) != null && this.conventionService.isNullOrEmpty(properties.get(key).toString())) { continue; } FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().equals(key.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null); if (currentFieldSet == null) { logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set " + key + " not found for comment " + properties.get(key).toString()); continue; } this.addSimpleCommentField(propertyDefinitionEntity, currentFieldSet, properties.get(key).toString()); } else { if (!key.toLowerCase(Locale.ROOT).trim().startsWith("multiple_")) { FieldSetEntity currentFieldSet = null; eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = null; for (FieldSetEntity fieldSetEntity : fieldSetEntities) { List fieldEntities = fieldSetEntity.getFieldById(key.trim()); if (!fieldEntities.isEmpty()) { currentFieldSet = fieldSetEntity; currentField = fieldEntities.getFirst(); } } if (currentField == null) { logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set not found for field " + key + " " + properties.get(key).toString()); continue; } this.addSimpleField(item, propertyDefinitionEntity, currentFieldSet, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap); } } } this.addMultipleField(item, properties, descriptionTemplateDefinitionEntity, propertyDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap); for (PropertyDefinitionFieldSetEntity fieldSetEntity : propertyDefinitionEntity.getFieldSets().values()){ int newOrdinal = 0; for (PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity : fieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() >= 0).sorted(Comparator.comparingInt(PropertyDefinitionFieldSetItemEntity::getOrdinal)).toList()){ propertyDefinitionFieldSetItemEntity.setOrdinal(newOrdinal); newOrdinal++; } for (PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity : fieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() < 0).sorted(Comparator.comparingInt(PropertyDefinitionFieldSetItemEntity::getOrdinal)).toList()){ propertyDefinitionFieldSetItemEntity.setOrdinal(newOrdinal); newOrdinal++; } } return propertyDefinitionEntity; } private void addMultipleField(Dataset item, Map properties, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, PropertyDefinitionEntity propertyDefinitionEntity, Map referenceTypeDefinitionEntityMap, Map referenceMap){ Map groupOrdinalMapping = new HashMap<>(); for (String key : properties.keySet()) { if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) { String newKey = key.trim().substring("multiple_".length()); String[] keyParts = newKey.split("_"); if (keyParts.length > 4 && newKey.contains("_multiple_")){ String[] multiParts = newKey.split("_multiple_"); if (multiParts.length == 2){ newKey = multiParts[0] + "_" + multiParts[1].split("_")[multiParts[1].split("_").length - 1]; keyParts = newKey.split("_"); } } String fieldSetId = ""; String groupId = ""; int ordinal = Integer.MIN_VALUE; String fieldId = ""; if (keyParts.length == 4) { fieldSetId = keyParts[0].trim(); groupId = keyParts[1].trim(); fieldId = keyParts[3].trim(); } else if (keyParts.length == 3) { fieldSetId = keyParts[0].trim(); groupId = keyParts[1].trim(); fieldId = keyParts[2].trim(); } else { throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field group key has invalid format " + key + " " + properties.get(key).toString()); } try { if (keyParts.length == 4) ordinal = Integer.parseInt(keyParts[2].trim()); } catch (Exception e){ throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field group key has invalid format " + key + " " + properties.get(key).toString()); } eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = descriptionTemplateDefinitionEntity.getFieldById(fieldId).stream().findFirst().orElse(null); if (currentField == null) { logger.warn("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set not found for field " + key + " " + properties.get(key).toString()); continue; } PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(fieldSetId, null); if (propertyDefinitionFieldSetEntity == null) { propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); propertyDefinitionFieldSetEntity.setItems(new ArrayList<>()); propertyDefinitionEntity.getFieldSets().put(fieldId, propertyDefinitionFieldSetEntity); } if (ordinal == Integer.MIN_VALUE) { if (groupOrdinalMapping.containsKey(new MultipleFieldKey(fieldSetId, groupId))){ ordinal = groupOrdinalMapping.get(new MultipleFieldKey(fieldSetId, groupId)); } else { ordinal = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> !x.getFields().isEmpty()).map(PropertyDefinitionFieldSetItemEntity::getOrdinal).min(Comparator.comparing(x -> x)).orElse(-1) - 1; ordinal = Math.min(ordinal, -1); } } if (!groupOrdinalMapping.containsKey(new MultipleFieldKey(fieldSetId, groupId))){ groupOrdinalMapping.put(new MultipleFieldKey(fieldSetId, groupId), ordinal); } else { if (groupOrdinalMapping.get(new MultipleFieldKey(fieldSetId, groupId)) != ordinal) throw new MyApplicationException("Invalid multiple key ordinal " + key); } this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap); } } } private void addMultipleField(Dataset item, PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, Map referenceTypeDefinitionEntityMap, Map referenceMap){ PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null); if (propertyDefinitionFieldSetItemEntity == null){ propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity(); propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>()); propertyDefinitionFieldSetItemEntity.setOrdinal(ordinal); propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); } propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); } private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, Map referenceTypeDefinitionEntityMap, Map referenceMap){ PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null); if (propertyDefinitionFieldSetEntity == null) { propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); propertyDefinitionFieldSetEntity.setItems(new ArrayList<>()); propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity); } PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null; if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){ propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity(); propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>()); propertyDefinitionFieldSetItemEntity.setOrdinal(0); propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); } else { propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst(); } propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); } private void addSimpleCommentField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, String comment){ PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null); if (propertyDefinitionFieldSetEntity == null) { propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); propertyDefinitionFieldSetEntity.setItems(new ArrayList<>()); propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity); } PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null; if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){ propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity(); propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>()); propertyDefinitionFieldSetItemEntity.setOrdinal(0); propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); } else { propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst(); } propertyDefinitionFieldSetItemEntity.setComment(comment); } private FieldEntity buildField(Dataset item, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, Map referenceTypeDefinitionEntityMap, Map referenceMap){ FieldEntity fieldEntity = new FieldEntity(); String textValue = properties.get(currentField.getId()) != null ? properties.get(currentField.getId()).toString() : null; if (textValue == null || textValue.isEmpty()) return fieldEntity; switch (currentField.getData().getFieldType()){ case FREE_TEXT, TEXT_AREA, RICH_TEXT_AREA, RADIO_BOX -> fieldEntity.setTextValue(textValue.trim()); case CHECK_BOX, BOOLEAN_DECISION -> fieldEntity.setTextValue(textValue.trim().toLowerCase(Locale.ROOT)); case DATE_PICKER -> { Instant instant = null; if(!this.conventionService.isNullOrEmpty(textValue)) { try { instant = Instant.parse(textValue); } catch (DateTimeParseException ex) { instant = LocalDate.parse(textValue).atStartOfDay().toInstant(ZoneOffset.UTC); } } fieldEntity.setDateValue(instant); } case SELECT -> { if(!this.conventionService.isNullOrEmpty(textValue)) { String[] valuesParsed = migrationTools.tryParseJsonAsObjectString(String[].class, textValue); fieldEntity.setTextListValue(valuesParsed == null ? List.of(textValue) : Arrays.stream(valuesParsed).toList()); } } case DATASET_IDENTIFIER, VALIDATION -> { if(!this.conventionService.isNullOrEmpty(textValue)) { ExternalIdentifierEntity externalIdentifierEntity = migrationTools.tryParseJsonAsObjectString(ExternalIdentifierEntity.class, textValue); if (externalIdentifierEntity == null) throw new MyApplicationException("Could not parse dataset External Identifier : " + textValue); fieldEntity.setExternalIdentifier(externalIdentifierEntity); } } case UPLOAD -> { if(!this.conventionService.isNullOrEmpty(textValue)) { Map valuesParsed = migrationTools.tryParseJsonAsObjectString(Map.class, textValue); if (valuesParsed == null) { valuesParsed = (Map) properties.get(currentField.getId()); } if (valuesParsed == null) throw new MyApplicationException("Could not parse upload : " + textValue); String id = valuesParsed.getOrDefault("id", null); if (id == null) throw new MyApplicationException("Could not parse upload : " + textValue); fieldEntity.setTextValue(UUID.fromString(id).toString()); } } case CURRENCY -> { if(!this.conventionService.isNullOrEmpty(textValue)) { Currency currency = migrationTools.tryParseJsonAsObjectString(Currency.class, textValue); if (currency == null && textValue.toUpperCase(Locale.ROOT).contains("EUR")) { currency = new Currency(); currency.setName("Euro"); currency.setValue("EUR"); } if (currency == null && textValue.toUpperCase(Locale.ROOT).contains("US DOLLAR")) { currency = new Currency(); currency.setName("US DOLLAR"); currency.setValue("USD"); } if (currency == null) throw new MyApplicationException("Could not parse Currency : " + textValue); //TODO: {"name":"Euro","value":"EUR"} what we want to keep ? fieldEntity.setTextValue(currency.getName()); } } case TAGS -> { if(!this.conventionService.isNullOrEmpty(textValue)) { Tag[] tags = migrationTools.tryParseJsonAsObjectString(Tag[].class, textValue); if (tags == null) { Tag tag = migrationTools.tryParseJsonAsObjectString(Tag.class, textValue); if (tag != null) tags = List.of(tag).toArray(Tag[]::new); } if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, migrationTools.cleanTagString(textValue)); if (tags == null) throw new MyApplicationException("Could not parse tag : " + textValue); fieldEntity.setTextListValue(Arrays.stream(tags).toList().stream().map(Tag::getName).toList()); } } case INTERNAL_ENTRIES_DMPS -> throw new MyApplicationException("Found INTERNAL_ENTRIES_DMPS into description"); case INTERNAL_ENTRIES_DESCRIPTIONS -> throw new MyApplicationException("Found INTERNAL_ENTRIES_DMPS into description"); case REFERENCE_TYPES -> { if(!this.conventionService.isNullOrEmpty(textValue)) { //TODO Map[] references = migrationTools.tryParseJsonAsObjectString(Map[].class, textValue); ReferenceTypeDataEntity referenceTypeDataEntity = (ReferenceTypeDataEntity)currentField.getData(); if (referenceTypeDataEntity == null) throw new MyApplicationException("Could not parse description template reference : " + textValue); if (references == null) { Map reference = migrationTools.tryParseJsonAsObjectString(Map.class, textValue); if (reference != null) references = List.of(reference).toArray(Map[]::new); } if (references == null && !textValue.contains("{") && !textValue.contains("[") ) { if (!textValue.contains(",")) { Map ref = new HashMap<>(); ref.put("name", textValue); if (!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) { ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null); if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); ref.put("source", referenceTypeDefinition.getSources().getFirst().getKey()); } else if (ReferenceTypeIds.DataRepositories.equals(referenceTypeDataEntity.getReferenceTypeId())) { ref.put("source", "openaire"); } else if (ReferenceTypeIds.Services.equals(referenceTypeDataEntity.getReferenceTypeId())) { ref.put("source", "eosc"); } else if (ReferenceTypeIds.Registries.equals(referenceTypeDataEntity.getReferenceTypeId())) { ref.put("source", "rda-metadata-schemes"); } else if (ReferenceTypeIds.Organizations.equals(referenceTypeDataEntity.getReferenceTypeId())) { ref.put("source", "openaire"); } else if (ReferenceTypeIds.Datasets.equals(referenceTypeDataEntity.getReferenceTypeId())) { ref.put("source", "openaire"); } else if (ReferenceTypeIds.Journal.equals(referenceTypeDataEntity.getReferenceTypeId())) { ref.put("source", "openaire"); } else if (ReferenceTypeIds.License.equals(referenceTypeDataEntity.getReferenceTypeId())) { ref.put("source", "opendefinition"); } else if (ReferenceTypeIds.PubRepositories.equals(referenceTypeDataEntity.getReferenceTypeId())) { ref.put("source", "opendefinition"); } else if (ReferenceTypeIds.Researcher.equals(referenceTypeDataEntity.getReferenceTypeId())) { Matcher pattern = Pattern.compile("\\((.*?)\\)").matcher(textValue); if (pattern.find()) { String matchedString = pattern.group(0); ref.put("name", textValue.replace(matchedString, "").trim()); matchedString = matchedString.substring(1, matchedString.length() - 1); ref.put("source", matchedString.split(":")[0]); ref.put("id", matchedString.split(":")[1]); } else { ref.put("name", textValue); ref.put("source", "orcid"); } } if (ref.containsKey("source")) references = List.of(ref).toArray(new Map[1]); } else { String source = null; if (!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) { ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null); if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); //if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); source = referenceTypeDefinition.getSources().getFirst().getKey(); String[] parts = null; if (textValue.startsWith("[\"")){ parts = migrationTools.tryParseJsonAsObjectString(String[].class, textValue); } else{ if (textValue.startsWith("[")) textValue = textValue.substring(1).trim(); if (textValue.endsWith("]")) textValue = textValue.substring(0, textValue.length() -1).trim(); parts = textValue.split(","); } if (parts != null) { List> items = new ArrayList<>(); for (String part : parts) { if (this.conventionService.isNullOrEmpty(part)) continue; part = part.trim().replace(",", "").trim(); if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue; Map ref = new HashMap<>(); ref.put("name", part); ref.put("source", source); items.add(ref); } if (items.isEmpty()) break; references = items.toArray(new Map[items.size()]); } } } } if (references == null && !ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())){ if (textValue.equalsIgnoreCase("[\"dublin core\",\"{\\\"id\\\":\\\"metadataschema:rd-alliance/dublin-core\\\",\\\"label\\\":\\\"Dublin Core\\\",\\\"source\\\":\\\"\\\",\\\"uri\\\":\\\"http://dublincore.org\\\"}\"]")){ Map ref = new HashMap<>(); ref.put("name", "Dublin Core"); ref.put("source", "metadataschema"); ref.put("id", "rd-alliance/dublin-core"); ref.put("uri", "http://dublincore.org"); references = List.of(ref).toArray(new Map[1]); } else if (textValue.equalsIgnoreCase("[\"Datacite\",\"{\\\"id\\\":\\\"metadataschema:rd-alliance/datacite-metadata-schema\\\",\\\"label\\\":\\\"DataCite Metadata Schema\\\",\\\"source\\\":\\\"\\\",\\\"uri\\\":\\\"http://schema.datacite.org\\\"}\"]")){ Map ref = new HashMap<>(); ref.put("name", "DataCite Metadata Schema"); ref.put("source", "metadataschema"); ref.put("id", "rd-alliance/datacite-metadata-schema"); ref.put("uri", "http://schema.datacite.org"); references = List.of(ref).toArray(new Map[1]); } else if (textValue.equalsIgnoreCase("[\" .\"]")) { break; } else if (textValue.equalsIgnoreCase("[INT]")) { ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null); if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); //if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); Map ref = new HashMap<>(); ref.put("name", "INT"); ref.put("source", referenceTypeDefinition.getSources().getFirst().getKey()); references = List.of(ref).toArray(new Map[1]); } } if (references == null && ReferenceTypeIds.Researcher.equals(referenceTypeDataEntity.getReferenceTypeId())){ if (textValue.equals("[\"-\"]")){ break; } else if (textValue.startsWith("[\"") && !textValue.startsWith("[\"{") && textValue.contains(",")){ references = migrationTools.tryParseJsonAsObjectString(Map[].class, textValue.replace(textValue.substring(1, textValue.indexOf(",") + 1), "")); } else if (textValue.contains(",") && !textValue.contains("{") && !textValue.contains("[") ){ String[] parts = textValue.split(","); List> items = new ArrayList<>(); for (String part : parts){ if (this.conventionService.isNullOrEmpty(part)) continue; part = part.trim().replace(",", "").trim(); if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue; Matcher pattern = Pattern.compile("\\((.*?)\\)").matcher(part); Map ref = new HashMap<>(); if (pattern.find()){ String matchedString = pattern.group(0); ref.put("name", part.replace(matchedString, "").trim()); matchedString = matchedString.substring(1, matchedString.length() - 1); ref.put("source", matchedString.split(":")[0]); ref.put("id", matchedString.split(":")[1]); }else { ref.put("name", part); ref.put("source", "orcid"); } items.add(ref); } if (items.isEmpty()) break; references = items.toArray(new Map[items.size()]); } } if (references == null && (ReferenceTypeIds.Datasets.equals(referenceTypeDataEntity.getReferenceTypeId()) || ReferenceTypeIds.DataRepositories.equals(referenceTypeDataEntity.getReferenceTypeId()) || ReferenceTypeIds.Services.equals(referenceTypeDataEntity.getReferenceTypeId()) || ReferenceTypeIds.License.equals(referenceTypeDataEntity.getReferenceTypeId()))){ textValue = textValue.trim(); if (textValue.equals("[\".\"]")){ break; } else if (!textValue.contains("{")) { String[] parts = null; if (textValue.startsWith("[\"")){ parts = migrationTools.tryParseJsonAsObjectString(String[].class, textValue); } else{ if (textValue.startsWith("[")) textValue = textValue.substring(1).trim(); if (textValue.endsWith("]")) textValue = textValue.substring(0, textValue.length() -1).trim(); parts = textValue.split(","); } if (parts != null) { List> items = new ArrayList<>(); for (String part : parts) { if (this.conventionService.isNullOrEmpty(part)) continue; part = part.trim().replace(",", "").trim(); if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue; Map ref = new HashMap<>(); ref.put("name", part); if (ReferenceTypeIds.Services.equals(referenceTypeDataEntity.getReferenceTypeId())) ref.put("source", "eosc"); else if (ReferenceTypeIds.License.equals(referenceTypeDataEntity.getReferenceTypeId())) ref.put("source", "opendefinition"); else ref.put("source", "openaire"); items.add(ref); } if (items.isEmpty()) break; references = items.toArray(new Map[items.size()]); } } } if (references == null){ throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getCreated() + " " + item.getLabel() + " Could not parse reference : " + referenceTypeDataEntity.getReferenceTypeId() + " with value " + textValue); } fieldEntity.setTextListValue(new ArrayList<>()); for (Map reference : references){ ReferenceEntity referenceEntity = this.geReferenceEntity(item, reference, referenceTypeDefinitionEntityMap, referenceTypeDataEntity, referenceMap); if (referenceEntity != null) fieldEntity.getTextListValue().add(referenceEntity.getId().toString()); } } } } return fieldEntity; } private ReferenceEntity geReferenceEntity(Dataset dataset, Map item, Map referenceTypeDefinitionEntityMap, ReferenceTypeDataEntity referenceTypeDataEntity, Map referenceMap){ ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null); if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(item)); if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(item)); ReferenceEntity data = new ReferenceEntity(); if (item.containsKey("name")) data.setLabel((String) item.get("name")); if (this.conventionService.isNullOrEmpty(data.getLabel()) && item.containsKey("label")) data.setLabel((String) item.get("label")); if (item.containsKey("abbreviation")) data.setAbbreviation((String) item.get("abbreviation")); if (item.containsKey("pid")) data.setReference((String)item.get("pid")); if (item.containsKey("key")) data.setSource((String)item.get("key")); if (this.conventionService.isNullOrEmpty(data.getSource()) && item.containsKey("source")) data.setSource((String) item.get("source")); if (item.containsKey("reference") && !this.conventionService.isNullOrEmpty(((String)item.get("reference")))){ if (((String)item.get("reference")).contains(":")) { String[] referenceParts = ((String)item.get("reference")).split(":", 2); if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(referenceParts[1]); if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource(referenceParts[0]); } else if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(((String)item.get("reference"))); } if (item.containsKey("tag") && !this.conventionService.isNullOrEmpty(((String)item.get("tag")))){ if (((String)item.get("tag")).equalsIgnoreCase("openaire")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); } else if (((String)item.get("tag")).equalsIgnoreCase("Open Definition")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("opendefinition"); } else if (((String)item.get("tag")).equalsIgnoreCase("RDA Metadata Schemes")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("rda-metadata-schemes"); } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE publication repositories 1")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE publication repositories 2")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE publication repositories 3")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE Journals")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE Alternative")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); }else if (((String)item.get("tag")).equalsIgnoreCase("Taxonomies")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("taxonomy"); } else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE Publications")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire"); } else if (((String)item.get("tag")).equalsIgnoreCase("EOSC Providers")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("eosc"); } else if (((String)item.get("tag")).equalsIgnoreCase("ORCID")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("orcid"); } else if (((String)item.get("tag")).equalsIgnoreCase("Zenodo")) { if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("zenodo"); } } if (this.conventionService.isNullOrEmpty(data.getReference()) && item.containsKey("id") && !this.conventionService.isNullOrEmpty(((String)item.get("id")))) { if (((String)item.get("id")).startsWith("datarepo:re3data/")) { if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(((String)item.get("id")).substring("datarepo:re3data/".length())); if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("re3data"); } else if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(((String)item.get("id"))); data.setReference((String)item.get("id")); } data.setId(UUID.randomUUID()); data.setCreatedAt(Instant.now()); data.setUpdatedAt(Instant.now()); data.setSourceType(ReferenceSourceType.External); data.setTypeId(referenceTypeDataEntity.getReferenceTypeId()); data.setIsActive(IsActive.Active); eu.eudat.commons.types.reference.DefinitionEntity definitionEntity = new eu.eudat.commons.types.reference.DefinitionEntity(); definitionEntity.setFields(new ArrayList<>()); for (Map.Entry entries : item.entrySet()){ if (entries.getValue() != null && !entries.getKey().equals("name") && !entries.getKey().equals("label")&& !entries.getKey().equals("abbreviation") && !entries.getKey().equals("id") && !entries.getKey().equals("key") && !entries.getKey().equals("pid")&& !entries.getKey().equals("source")) { eu.eudat.commons.types.reference.FieldEntity field = new eu.eudat.commons.types.reference.FieldEntity(); field.setCode(entries.getKey()); field.setDataType(ReferenceFieldDataType.Text); field.setValue(entries.getValue().toString()); definitionEntity.getFields().add(field); } } if (!this.conventionService.isListNullOrEmpty(definitionEntity.getFields())) data.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity)); if (this.conventionService.isNullOrEmpty(data.getLabel())) data.setLabel(data.getReference()); if (this.conventionService.isNullOrEmpty(data.getSource()) && referenceTypeDefinition.getSources().size() == 1 && !ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) { data.setSource(referenceTypeDefinition.getSources().getFirst().getKey()); } if (this.conventionService.isNullOrEmpty(data.getLabel()) || data.getLabel().equals("null")){ logger.warn("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " no label for the reference : " + referenceTypeDataEntity.getReferenceTypeId() + " " + this.jsonHandlingService.toJsonSafe(item)); //TODO return null; } if (this.conventionService.isNullOrEmpty(data.getSource())) throw new MyApplicationException("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " Source not found source for reference : " + referenceTypeDataEntity.getReferenceTypeId() + " " + this.jsonHandlingService.toJsonSafe(item)); //TODO if (this.conventionService.isNullOrEmpty(data.getReference())) { data.setReference(this.queryFactory.query(ReferenceQuery.class).sources(data.getSource()).like(data.getLabel()).typeIds(data.getTypeId()).sourceTypes(ReferenceSourceType.External).collectAs(new BaseFieldSet().ensure(Reference._reference, Reference._label)) .stream().filter(x-> x.getLabel().equals(data.getLabel())).map(ReferenceEntity::getReference).findFirst().orElse(null));//TODO } if (this.conventionService.isNullOrEmpty(data.getReference())) { if (referenceTypeDefinition.getSources().stream().filter(x-> x.getType().equals(ExternalFetcherSourceType.API)).map(x-> (ExternalFetcherApiSourceConfigurationEntity)x).anyMatch(x-> x.getUrl().toLowerCase(Locale.ROOT).contains("eestore.paas2.uninett.no"))){ if (data.getLabel().equalsIgnoreCase("Attribution 4.0 International")){ data.setReference("CC-BY-4.0"); } else if (data.getLabel().equalsIgnoreCase("Attribution-NonCommercial 4.0 International")){ data.setReference("CC-BY-NC-4.0"); } else if (data.getLabel().equalsIgnoreCase("Attribution-NonCommercial-NoDerivatives 4.0 International")){ data.setReference("CC-BY-NC-ND-4.0"); } else if (data.getLabel().equalsIgnoreCase("Attribution-NoDerivatives 4.0 International")){ data.setReference("CC-BY-ND-4.0"); } else if (data.getLabel().equalsIgnoreCase("Attribution-NonCommercial-ShareAlike 4.0 International") ){ data.setReference("CC-BY-NC-SA-4.0"); } } if (data.getTypeId().equals(ReferenceTypeIds.License) && data.getSource().equalsIgnoreCase("OpenAIRE")){ if (data.getLabel().equalsIgnoreCase("Academic Free License 3.0")){ data.setReference("AFL-3.0"); } else if (data.getLabel().equalsIgnoreCase("Attribution Assurance Licenses")){ data.setReference("AFL-3.0"); } } else if (data.getTypeId().equals(ReferenceTypeIds.Datasets) && data.getSource().equalsIgnoreCase("OpenAIRE")) { if (data.getLabel().equalsIgnoreCase("Bulk RNA-seq U2OS cells treated with small molecules") && data.getSource().equalsIgnoreCase("OpenAIRE")){ data.setReference("50|_____OmicsDI::7fc1bcd133082463ec1948b82f1ad0ee"); } else if (data.getLabel().equalsIgnoreCase("Genome-wide effect of the microenvironment over the Galectin-7 regulation network in HeLa and SiHa tumors in nude mice.")){ data.setReference("50|_____OmicsDI::4e0de948bf961b96dbdef82797b5a8dc"); } // else if (data.getLabel().equalsIgnoreCase("other")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("none")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("to be clarified later")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("articles")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Freistilringen")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Randomized Phase 3 Trial of ALIMTA (Pemetrexed) and Carboplatin versus Etoposide and Carboplatin in Extensive-Stage Small Cell Lung Cancer")){ data.setReference("50|datacite____::1df6f72ab95712d5f26ecb43b2d1d303"); } } else if (data.getTypeId().equals(ReferenceTypeIds.DataRepositories) && data.getSource().equalsIgnoreCase("OpenAIRE")) { if (data.getLabel().equalsIgnoreCase("INPTDAT")){ data.setReference("re3data_____::r3d100013120"); } if (data.getLabel().equalsIgnoreCase("OpenAPC Global Initiative")){ data.setReference("apc_________::openapc"); } else if (data.getLabel().equalsIgnoreCase("Human Gene and Protein Database")){ data.setReference("fairsharing_::1592"); } else if (data.getLabel().equalsIgnoreCase("Bacterial protein tYrosine Kinase database")){ data.setReference("fairsharing_::1557"); } // else if (data.getLabel().equalsIgnoreCase("other")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Zenodo")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Zenado")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("none")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("articles")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("https://github.com/sisyphe-re/riot_rpl_udp_scenarios")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else else if (data.getLabel().equalsIgnoreCase("Cool data")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Raw data")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Repository hosted by Zenodo")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("ICPSR - Interuniversity Consortium for Political and Social Research")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("XC7 pharmaceutical development IPF")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("nct")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("UK Data Service")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("figshare")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("University of Strathclyde")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } } else if (data.getTypeId().equals(ReferenceTypeIds.Services) && data.getSource().equalsIgnoreCase("eosc")) { // if (data.getLabel().equalsIgnoreCase("website")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("integration1")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("SeaDataNet Common Data Index (CDI)")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Europeana APIs")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Imaging at Plentzia Marine Station-UPV/EHU")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Spectroscopy")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Library services at Stazione Zoologica Anton Dohrn")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("PRACE Training Portal")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("DARIAH Science Gateway")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Finding Anisotropy Tensor")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Imaging at University of Bergen-SLRC")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Training facilities at Toralla Marine Science Station - Vigo University Marine Research Centre")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Zenodo")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Digital Commons")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } }else if (data.getTypeId().equals(ReferenceTypeIds.Registries) && data.getSource().equalsIgnoreCase("rda-metadata-schemes")) { // if (data.getLabel().equalsIgnoreCase("DataCite Metadata Schema")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("AgMES (Agricultural Metadata Element Set)")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } } else if (data.getTypeId().equals(ReferenceTypeIds.Organizations) && data.getSource().equalsIgnoreCase("OpenAIRE")) { if (data.getLabel().equalsIgnoreCase("Programa de Pós-Graduação em Ciência da Informação Ibict/UFRJ")){ data.setReference("pending_org_::f20d38c291e7d4a7a11d08481eebac44"); } else if (data.getLabel().equalsIgnoreCase("Applied Decision Science (United States)")){ data.setReference("openorgs____::0000093299"); } else if (data.getLabel().equalsIgnoreCase("BIODOL THERAPEUTICS")){ data.setReference("pending_org_::4c550b2917a6fc395bdb79eccdafe950"); } else if (data.getLabel().equalsIgnoreCase("OKKO")){ data.setReference("pending_org_::37ddf98c5fd2d6373fd315b0371fd4ac"); } else if (data.getLabel().equalsIgnoreCase("Health Research Board")){ data.setReference("openorgs____::0000006166"); } else if (data.getLabel().equalsIgnoreCase("Vidzeme University of Applied Sciences")){ data.setReference("openorgs____::0000029719"); } // else if (data.getLabel().equalsIgnoreCase("Other Lab")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("ATS APPLIED TECH SYSTEMS LTD")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } else if (data.getLabel().equalsIgnoreCase("Universidade Nova de Lisboa Centro de Linguística")){ // data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); // data.setSourceType(ReferenceSourceType.Internal); // if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); // } } else if (data.getTypeId().equals(ReferenceTypeIds.Researcher)){ if (data.getLabel().equalsIgnoreCase("Radosław Cichocki")){ data.setReference("0000-0002-4667-8445"); } else if (data.getLabel().equalsIgnoreCase("Kiranmai Uppuluri")){ data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); data.setSourceType(ReferenceSourceType.Internal); if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); } } } if (this.conventionService.isNullOrEmpty(data.getReference())) { data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT)); data.setSourceType(ReferenceSourceType.Internal); data.setSource("Internal"); if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); logger.warn("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " reference auto created as internal " + referenceTypeDataEntity.getReferenceTypeId() + " for value " + this.jsonHandlingService.toJsonSafe(item)); } ReferenceEntity existingEntity = referenceMap.getOrDefault(new ReferenceKey(data), null); if(existingEntity == null){ if (data.getCreatedById() != null) referenceMap.put(new ReferenceKey(data), data); this.entityManager.persist(data); return data; } else { return existingEntity; } } public static class Currency{ private String name; private String value; public String getName() { return name; } public void setName(String name) { this.name = name; } public String getValue() { return value; } public void setValue(String value) { this.value = value; } } public static class Tag{ private String id; private String name; public String getId() { return id; } public void setId(String id) { this.id = id; } public String getName() { return name; } public void setName(String name) { this.name = name; } } public static class MultipleFieldKey{ private final String fieldSetId; private final String groupKey; private final int hashCode; public MultipleFieldKey(String fieldSetId, String groupKey) { this.fieldSetId = fieldSetId; this.groupKey = groupKey; this.hashCode = Objects.hash(this.fieldSetId, this.groupKey); } public String getFieldSetId() { return fieldSetId; } public String getGroupKey() { return groupKey; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; MultipleFieldKey that = (MultipleFieldKey) o; return Objects.equals(fieldSetId, that.getFieldSetId()) && Objects.equals(groupKey, that.getGroupKey()); } @Override public int hashCode() { return this.hashCode; } } public static class ReferenceKey { private final UUID type; private final String source; private final String reference; private final int hashCode; public ReferenceKey(ReferenceEntity reference) { this.type = reference.getTypeId(); this.source = reference.getSource().trim(); this.reference = reference.getReference().trim(); hashCode = Objects.hash(this.type, this.source, this.reference); } public UUID getType() { return type; } public String getSource() { return source; } public String getReference() { return reference; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ReferenceKey that = (ReferenceKey) o; return Objects.equals(type, that.getType()) && Objects.equals(source, that.getSource()) && Objects.equals(reference, that.getReference()); } @Override public int hashCode() { return this.hashCode; } } }