MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

migration changes

This commit is contained in:
Efstratios Giannopoulos 2024-02-28 18:48:46 +02:00
parent 0e7813cded
commit 8237e7936d
17 changed files with 810 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dmp-backend/core/src/main/java/eu/eudat/service/externalfetcher/ExternalFetcherServiceImpl.java
View File

@ -15,6 +15,9 @@ import eu.eudat.service.externalfetcher.config.entities.*;
import eu.eudat.service.externalfetcher.models.ExternalDataResult;
import eu.eudat.service.externalfetcher.criteria.ExternalReferenceCriteria;
import gr.cite.tools.exception.MyApplicationException;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import net.minidev.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -27,6 +30,7 @@ import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import javax.net.ssl.SSLException;
import java.util.*;
import java.util.stream.Collectors;
@ -60,7 +64,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
List<SourceBaseConfiguration> apiSourcesToUse = sources;
if (!this.conventionService.isNullOrEmpty(key)){
apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).toList();
apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).collect(Collectors.toList());
}
if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult();

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DataRepositoryMigrationService.java
View File

@ -53,7 +53,7 @@ public class DataRepositoryMigrationService {
entityManager.detach(item);
if (item.getReference() == null || !item.getReference().contains(":")){
logger.warn("Reference generated because is null DataRepository " + item.getId());
item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT));
item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
}
eu.old.eudat.models.data.datarepository.DataRepositoryModel model = new eu.old.eudat.models.data.datarepository.DataRepositoryModel().fromDataModel(item);

690
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -2,15 +2,12 @@ package eu.old.eudat.migration;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.DescriptionStatus;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.*;
import eu.eudat.commons.types.description.*;
import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity;
import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity;
import eu.eudat.commons.types.dmpblueprint.DefinitionEntity;
import eu.eudat.commons.types.externalfetcher.ExternalFetcherBaseSourceConfigurationEntity;
import eu.eudat.commons.types.externalfetcher.ExternalFetcherApiSourceConfigurationEntity;
import eu.eudat.commons.types.referencetype.ReferenceTypeDefinitionEntity;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.*;
@ -18,16 +15,17 @@ import eu.eudat.model.Dmp;
import eu.eudat.model.Reference;
import eu.eudat.model.ReferenceType;
import eu.eudat.query.*;
import eu.eudat.service.reference.ReferenceService;
import eu.old.eudat.data.dao.entities.DatasetDao;
import eu.old.eudat.data.entities.Dataset;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import eu.old.eudat.models.data.user.components.datasetprofile.Field;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException;
import org.apache.commons.lang3.StringEscapeUtils;
import org.json.JSONObject;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
@ -40,6 +38,8 @@ import java.time.LocalDate;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Service
@ -54,14 +54,16 @@ public class DatasetMigrationService {
private static final boolean TestMode = false;
private final EntityManager entityManager;
private final ConventionService conventionService;
private final ReferenceService referenceService;
public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager, ConventionService conventionService) {
public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager, ConventionService conventionService, ReferenceService referenceService) {
this.databaseRepository = databaseRepository;
this.jsonHandlingService = jsonHandlingService;
this.queryFactory = queryFactory;
this.xmlHandlingService = xmlHandlingService;
this.entityManager = entityManager;
this.conventionService = conventionService;
this.referenceService = referenceService;
}
public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
@ -97,7 +99,7 @@ public class DatasetMigrationService {
List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities = this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collect();
List<DescriptionTemplateEntity> descriptionTemplates = this.queryFactory.query(DescriptionTemplateQuery.class).ids(items.stream().map(x-> x.getProfile().getId()).distinct().toList()).collect();
Map<UUID, UUID> creatorsByDmp = this.queryFactory.query(DmpQuery.class).ids(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collectAs(new BaseFieldSet().ensure(Dmp._id).ensure(Dmp._creator)).stream().collect(Collectors.toMap(DmpEntity::getId, DmpEntity::getCreatorId));;
Map<UUID, UUID> creatorsByDmp = this.queryFactory.query(DmpQuery.class).ids(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collectAs(new BaseFieldSet().ensure(Dmp._id).ensure(Dmp._creator)).stream().collect(Collectors.toMap(DmpEntity::getId, DmpEntity::getCreatorId));
Map<UUID, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity> descriptionTemplateDefinitionMap = new HashMap<>();
for (DescriptionTemplateEntity descriptionTemplateEntity : descriptionTemplates){
@ -107,20 +109,17 @@ public class DatasetMigrationService {
DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null);
if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) {
logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
}
UUID sectionId = definition.getSections().get(item.getDmpSectionIndex()).getId();
if (sectionId == null) {
logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
}
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = this.getOrCreateDmpDescriptionTemplateEntity(item, sectionId, dmpDescriptionTemplateEntities);
if (itemDescriptionTemplates.size() > 1) {
logger.error("Migrate Dataset " + item.getId() + " multiple DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
//TODO CLEAN Duplicates dmp, DescriptionTemplate, SectionId
throw new MyApplicationException("Migrate Dataset " + item.getId() + " multiple DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
}
DescriptionEntity data = new DescriptionEntity();
@ -151,7 +150,6 @@ public class DatasetMigrationService {
data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap)));
if (data.getCreatedById() == null){
logger.error("Migration skipped creator not found " + item.getId());
throw new MyApplicationException("Migration skipped creator not found " + item.getId());
}
@ -162,13 +160,12 @@ public class DatasetMigrationService {
page++;
}
} while (items != null && !items.isEmpty() && !TestMode);
throw new MyApplicationException("");
}
private List<DmpDescriptionTemplateEntity> getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities){
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
if (itemDescriptionTemplates.isEmpty()) {
logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
logger.warn("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
if (dmpDescriptionTemplateEntities.stream().anyMatch(x -> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId))) {
DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity();
dmpDescriptionTemplateEntity.setId(UUID.randomUUID());
@ -181,7 +178,7 @@ public class DatasetMigrationService {
this.entityManager.persist(dmpDescriptionTemplateEntity);
itemDescriptionTemplates = List.of(dmpDescriptionTemplateEntity);
} else {
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getLabel() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
}
}
return itemDescriptionTemplates;
@ -203,7 +200,7 @@ public class DatasetMigrationService {
FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().equals(key.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null);
if (currentFieldSet == null) {
logger.error("Field set " + key + " not found for comment " + properties.get(key).toString());
logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set " + key + " not found for comment " + properties.get(key).toString());
continue;
}
@ -221,16 +218,16 @@ public class DatasetMigrationService {
}
if (currentField == null) {
logger.error("Field set not found for field " + key + " " + properties.get(key).toString());
logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set not found for field " + key + " " + properties.get(key).toString());
continue;
}
this.addSimpleField(propertyDefinitionEntity, currentFieldSet, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap);
this.addSimpleField(item, propertyDefinitionEntity, currentFieldSet, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap);
}
}
}
this.addMultipleField(properties, descriptionTemplateDefinitionEntity, propertyDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap);
this.addMultipleField(item, properties, descriptionTemplateDefinitionEntity, propertyDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap);
for (PropertyDefinitionFieldSetEntity fieldSetEntity : propertyDefinitionEntity.getFieldSets().values()){
int newOrdinal = 0;
@ -246,12 +243,20 @@ public class DatasetMigrationService {
return propertyDefinitionEntity;
}
private void addMultipleField(Map<String, Object> properties, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, PropertyDefinitionEntity propertyDefinitionEntity, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
private void addMultipleField(Dataset item, Map<String, Object> properties, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, PropertyDefinitionEntity propertyDefinitionEntity, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
Map<MultipleFieldKey, Integer> groupOrdinalMapping = new HashMap<>();
for (String key : properties.keySet()) {
if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) {
String newKey = key.trim().substring("multiple_".length());
String[] keyParts = newKey.split("_", 4);
//newKey = newKey.replace("multiple_formats-to-plan-to-store_fa87dc73-d0b6-b707-c632-369f4191a306_", "");//TODO
String[] keyParts = newKey.split("_");
if (keyParts.length > 4 && newKey.contains("_multiple_")){
String[] multiParts = newKey.split("_multiple_");
if (multiParts.length == 2){
newKey = multiParts[0] + "_" + multiParts[1].split("_")[multiParts[1].split("_").length - 1];
keyParts = newKey.split("_");
}
}
String fieldSetId = "";
String groupId = "";
int ordinal = Integer.MIN_VALUE;
@ -266,19 +271,17 @@ public class DatasetMigrationService {
groupId = keyParts[1].trim();
fieldId = keyParts[2].trim();
} else {
logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString());
continue;
throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field group key has invalid format " + key + " " + properties.get(key).toString());
}
try {
if (keyParts.length == 4) ordinal = Integer.parseInt(keyParts[2].trim());
} catch (Exception e){
logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString());
continue;
throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field group key has invalid format " + key + " " + properties.get(key).toString());
}
eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = descriptionTemplateDefinitionEntity.getFieldById(fieldId).stream().findFirst().orElse(null);
if (currentField == null) {
logger.error("Field set not found for field " + key + " " + properties.get(key).toString());
logger.warn("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set not found for field " + key + " " + properties.get(key).toString());
continue;
}
@ -303,13 +306,13 @@ public class DatasetMigrationService {
if (groupOrdinalMapping.get(new MultipleFieldKey(fieldSetId, groupId)) != ordinal) throw new MyApplicationException("Invalid multiple key ordinal " + key);
}
this.addMultipleField(propertyDefinitionFieldSetEntity, ordinal, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap);
this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap);
}
}
}
private void addMultipleField(PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
private void addMultipleField(Dataset item, PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null);
if (propertyDefinitionFieldSetItemEntity == null){
@ -319,10 +322,10 @@ public class DatasetMigrationService {
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
}
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
}
private void addSimpleField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null);
if (propertyDefinitionFieldSetEntity == null) {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
@ -340,7 +343,7 @@ public class DatasetMigrationService {
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
}
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
}
@ -365,7 +368,7 @@ public class DatasetMigrationService {
propertyDefinitionFieldSetItemEntity.setComment(comment);
}
private FieldEntity buildField(eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
private FieldEntity buildField(Dataset item, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
FieldEntity fieldEntity = new FieldEntity();
String textValue = properties.get(currentField.getId()) != null ? properties.get(currentField.getId()).toString() : null;
if (textValue == null || textValue.isEmpty()) return fieldEntity;
@ -385,26 +388,23 @@ public class DatasetMigrationService {
}
case SELECT -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
String[] valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, textValue);
if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, this.cleanAsObjectString(textValue));
String[] valuesParsed = this.tryParseJsonAsObjectString(String[].class, textValue);
fieldEntity.setTextListValue(valuesParsed == null ? List.of(textValue) : Arrays.stream(valuesParsed).toList());
}
}
case DATASET_IDENTIFIER, VALIDATION -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
ExternalIdentifierEntity externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, textValue);
if (externalIdentifierEntity == null) externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, this.cleanAsObjectString(textValue));
ExternalIdentifierEntity externalIdentifierEntity = this.tryParseJsonAsObjectString(ExternalIdentifierEntity.class, textValue);
if (externalIdentifierEntity == null) throw new MyApplicationException("Could not parse dataset External Identifier : " + textValue);
fieldEntity.setExternalIdentifier(externalIdentifierEntity);
}
}
case UPLOAD -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
Map<String, String> valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, textValue);
Map<String, String> valuesParsed = this.tryParseJsonAsObjectString(Map.class, textValue);
if (valuesParsed == null) {
valuesParsed = (Map<String, String>) properties.get(currentField.getId());
}
if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, this.cleanAsObjectString(textValue));
if (valuesParsed == null) throw new MyApplicationException("Could not parse upload : " + textValue);
String id = valuesParsed.getOrDefault("id", null);
if (id == null) throw new MyApplicationException("Could not parse upload : " + textValue);
@ -413,8 +413,7 @@ public class DatasetMigrationService {
}
case CURRENCY -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
Currency currency = this.jsonHandlingService.fromJsonSafe(Currency.class, textValue);
if (currency == null) currency = this.jsonHandlingService.fromJsonSafe(Currency.class, this.cleanAsObjectString(textValue));
Currency currency = this.tryParseJsonAsObjectString(Currency.class, textValue);
if (currency == null && textValue.toUpperCase(Locale.ROOT).contains("EUR")) {
currency = new Currency();
currency.setName("Euro");
@ -433,18 +432,13 @@ public class DatasetMigrationService {
}
case TAGS -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
Tag[] tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, textValue);
if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, this.cleanAsObjectString(textValue));
Tag[] tags = this.tryParseJsonAsObjectString(Tag[].class, textValue);
if (tags == null) {
Tag tag = this.jsonHandlingService.fromJsonSafe(Tag.class, textValue);
if (tag == null) tag = this.jsonHandlingService.fromJsonSafe(Tag.class, this.cleanAsObjectString(textValue));
Tag tag = this.tryParseJsonAsObjectString(Tag.class, textValue);
if (tag != null) tags = List.of(tag).toArray(Tag[]::new);
}
if (tags == null) {
//throw new MyApplicationException("Could not parse tag : " + textValue); //TODO
logger.error("Could not parse tag : " + textValue);
break;
}
if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, this.cleanTagString(textValue));
if (tags == null) throw new MyApplicationException("Could not parse tag : " + textValue);
fieldEntity.setTextListValue(Arrays.stream(tags).toList().stream().map(Tag::getName).toList());
}
}
@ -453,25 +447,193 @@ public class DatasetMigrationService {
case INTERNAL_ENTRIES_DESCRIPTIONS -> throw new MyApplicationException("Found INTERNAL_ENTRIES_DMPS into description");
case REFERENCE_TYPES -> {
if(!this.conventionService.isNullOrEmpty(textValue)) { //TODO
Map<String, Object>[] references = this.jsonHandlingService.fromJsonSafe(Map[].class, textValue);
if (references == null) references = this.jsonHandlingService.fromJsonSafe(Map[].class, this.cleanAsObjectString(textValue));
if (references == null) {
Map<String, Object> reference = this.jsonHandlingService.fromJsonSafe(Map.class, textValue);
if (reference == null) reference = this.jsonHandlingService.fromJsonSafe(Map.class, this.cleanAsObjectString(textValue));
if (reference != null) references = List.of(reference).toArray(Map[]::new);
}
if (references == null){
logger.warn("Could not parse reference : " + textValue);
//throw new MyApplicationException("Could not parse reference : " + textValue);
break;
}
Map<String, Object>[] references = this.tryParseJsonAsObjectString(Map[].class, textValue);
ReferenceTypeDataEntity referenceTypeDataEntity = (ReferenceTypeDataEntity)currentField.getData();
if (referenceTypeDataEntity == null) throw new MyApplicationException("Could not parse description template reference : " + textValue);
if (references == null) {
Map<String, Object> reference = this.tryParseJsonAsObjectString(Map.class, textValue);
if (reference != null) references = List.of(reference).toArray(Map[]::new);
}
if (references == null && !textValue.contains("{") && !textValue.contains("[") ) {
if (!textValue.contains(",")) {
Map<String, Object> ref = new HashMap<>();
ref.put("name", textValue);
if (!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) {
ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null);
if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
ref.put("source", referenceTypeDefinition.getSources().getFirst().getKey());
} else if (ReferenceTypeIds.DataRepositories.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "openaire");
} else if (ReferenceTypeIds.Services.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "eosc");
} else if (ReferenceTypeIds.Registries.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "rda-metadata-schemes");
} else if (ReferenceTypeIds.Organizations.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "openaire");
} else if (ReferenceTypeIds.Datasets.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "openaire");
} else if (ReferenceTypeIds.Journal.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "openaire");
} else if (ReferenceTypeIds.License.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "opendefinition");
} else if (ReferenceTypeIds.PubRepositories.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "opendefinition");
} else if (ReferenceTypeIds.Researcher.equals(referenceTypeDataEntity.getReferenceTypeId())) {
Matcher pattern = Pattern.compile("\\((.*?)\\)").matcher(textValue);
if (pattern.find()) {
String matchedString = pattern.group(0);
ref.put("name", textValue.replace(matchedString, "").trim());
matchedString = matchedString.substring(1, matchedString.length() - 1);
ref.put("source", matchedString.split(":")[0]);
ref.put("id", matchedString.split(":")[1]);
} else {
ref.put("name", textValue);
ref.put("source", "orcid");
}
}
if (ref.containsKey("source")) references = List.of(ref).toArray(new Map[1]);
} else {
String source = null;
if (!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) {
ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null);
if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
//if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
source = referenceTypeDefinition.getSources().getFirst().getKey();
String[] parts = null;
if (textValue.startsWith("[\"")){
parts = this.tryParseJsonAsObjectString(String[].class, textValue);
} else{
if (textValue.startsWith("[")) textValue = textValue.substring(1).trim();
if (textValue.endsWith("]")) textValue = textValue.substring(0, textValue.length() -1).trim();
parts = textValue.split(",");
}
if (parts != null) {
List<Map<String, Object>> items = new ArrayList<>();
for (String part : parts) {
if (this.conventionService.isNullOrEmpty(part)) continue;
part = part.trim().replace(",", "").trim();
if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue;
Map<String, Object> ref = new HashMap<>();
ref.put("name", part);
ref.put("source", source);
items.add(ref);
}
if (items.isEmpty()) break;
references = items.toArray(new Map[items.size()]);
}
}
}
}
if (references == null && !ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())){
if (textValue.equalsIgnoreCase("[\"dublin core\",\"{\\\"id\\\":\\\"metadataschema:rd-alliance/dublin-core\\\",\\\"label\\\":\\\"Dublin Core\\\",\\\"source\\\":\\\"\\\",\\\"uri\\\":\\\"http://dublincore.org\\\"}\"]")){
Map<String, Object> ref = new HashMap<>();
ref.put("name", "Dublin Core");
ref.put("source", "metadataschema");
ref.put("id", "rd-alliance/dublin-core");
ref.put("uri", "http://dublincore.org");
references = List.of(ref).toArray(new Map[1]);
} else if (textValue.equalsIgnoreCase("[\"Datacite\",\"{\\\"id\\\":\\\"metadataschema:rd-alliance/datacite-metadata-schema\\\",\\\"label\\\":\\\"DataCite Metadata Schema\\\",\\\"source\\\":\\\"\\\",\\\"uri\\\":\\\"http://schema.datacite.org\\\"}\"]")){
Map<String, Object> ref = new HashMap<>();
ref.put("name", "DataCite Metadata Schema");
ref.put("source", "metadataschema");
ref.put("id", "rd-alliance/datacite-metadata-schema");
ref.put("uri", "http://schema.datacite.org");
references = List.of(ref).toArray(new Map[1]);
} else if (textValue.equalsIgnoreCase("[\" .\"]")) {
break;
} else if (textValue.equalsIgnoreCase("[INT]")) {
ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null);
if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
//if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
Map<String, Object> ref = new HashMap<>();
ref.put("name", "INT");
ref.put("source", referenceTypeDefinition.getSources().getFirst().getKey());
references = List.of(ref).toArray(new Map[1]);
}
}
if (references == null && ReferenceTypeIds.Researcher.equals(referenceTypeDataEntity.getReferenceTypeId())){
if (textValue.equals("[\"-\"]")){
break;
} else if (textValue.startsWith("[\"") && !textValue.startsWith("[\"{") && textValue.contains(",")){
references = this.tryParseJsonAsObjectString(Map[].class, textValue.replace(textValue.substring(1, textValue.indexOf(",") + 1), ""));
} else if (textValue.contains(",") && !textValue.contains("{") && !textValue.contains("[") ){
String[] parts = textValue.split(",");
List<Map<String, Object>> items = new ArrayList<>();
for (String part : parts){
if (this.conventionService.isNullOrEmpty(part)) continue;
part = part.trim().replace(",", "").trim();
if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue;
Matcher pattern = Pattern.compile("\\((.*?)\\)").matcher(part);
Map<String, Object> ref = new HashMap<>();
if (pattern.find()){
String matchedString = pattern.group(0);
ref.put("name", part.replace(matchedString, "").trim());
matchedString = matchedString.substring(1, matchedString.length() - 1);
ref.put("source", matchedString.split(":")[0]);
ref.put("id", matchedString.split(":")[1]);
}else {
ref.put("name", part);
ref.put("source", "orcid");
}
items.add(ref);
}
if (items.isEmpty()) break;
references = items.toArray(new Map[items.size()]);
}
}
if (references == null && (ReferenceTypeIds.Datasets.equals(referenceTypeDataEntity.getReferenceTypeId())
|| ReferenceTypeIds.DataRepositories.equals(referenceTypeDataEntity.getReferenceTypeId())
|| ReferenceTypeIds.Services.equals(referenceTypeDataEntity.getReferenceTypeId())
|| ReferenceTypeIds.License.equals(referenceTypeDataEntity.getReferenceTypeId()))){
textValue = textValue.trim();
if (textValue.equals("[\".\"]")){
break;
} else if (!textValue.contains("{")) {
String[] parts = null;
if (textValue.startsWith("[\"")){
parts = this.tryParseJsonAsObjectString(String[].class, textValue);
} else{
if (textValue.startsWith("[")) textValue = textValue.substring(1).trim();
if (textValue.endsWith("]")) textValue = textValue.substring(0, textValue.length() -1).trim();
parts = textValue.split(",");
}
if (parts != null) {
List<Map<String, Object>> items = new ArrayList<>();
for (String part : parts) {
if (this.conventionService.isNullOrEmpty(part)) continue;
part = part.trim().replace(",", "").trim();
if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue;
Map<String, Object> ref = new HashMap<>();
ref.put("name", part);
if (ReferenceTypeIds.Services.equals(referenceTypeDataEntity.getReferenceTypeId())) ref.put("source", "eosc");
else if (ReferenceTypeIds.License.equals(referenceTypeDataEntity.getReferenceTypeId())) ref.put("source", "opendefinition");
else ref.put("source", "openaire");
items.add(ref);
}
if (items.isEmpty()) break;
references = items.toArray(new Map[items.size()]);
}
}
}
if (references == null){
throw new MyApplicationException("Migrate Dataset " + item.getId() + " " + item.getCreated() + " " + item.getLabel() + " Could not parse reference : " + referenceTypeDataEntity.getReferenceTypeId() + " with value " + textValue);
}
fieldEntity.setTextListValue(new ArrayList<>());
for (Map<String, Object> item : references){
ReferenceEntity referenceEntity = this.geReferenceEntity(item, referenceTypeDefinitionEntityMap, referenceTypeDataEntity, textValue, referenceMap);
fieldEntity.getTextListValue().add(referenceEntity.getId().toString());
for (Map<String, Object> reference : references){
ReferenceEntity referenceEntity = this.geReferenceEntity(item, reference, referenceTypeDefinitionEntityMap, referenceTypeDataEntity, referenceMap);
if (referenceEntity != null) fieldEntity.getTextListValue().add(referenceEntity.getId().toString());
}
}
}
@ -479,18 +641,65 @@ public class DatasetMigrationService {
return fieldEntity;
}
private ReferenceEntity geReferenceEntity(Map<String, Object> item, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, ReferenceTypeDataEntity referenceTypeDataEntity, String textValue, Map<ReferenceKey, ReferenceEntity> referenceMap){
private ReferenceEntity geReferenceEntity(Dataset dataset, Map<String, Object> item, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, ReferenceTypeDataEntity referenceTypeDataEntity, Map<ReferenceKey, ReferenceEntity> referenceMap){
ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null);
if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + textValue);
if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + textValue);
if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(item));
if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(item));
ReferenceEntity data = new ReferenceEntity();
if (item.containsKey("name")) data.setLabel((String) item.get("name"));
if (this.conventionService.isNullOrEmpty(data.getLabel()) && item.containsKey("label")) data.setLabel((String) item.get("label"));
if (item.containsKey("abbreviation")) data.setAbbreviation((String) item.get("abbreviation"));
if (item.containsKey("id")) data.setReference((String)item.get("id"));
if (item.containsKey("reference")) data.setReference((String)item.get("reference"));
if (this.conventionService.isNullOrEmpty(data.getReference()) && item.containsKey("pid")) data.setReference((String)item.get("pid"));
if (item.containsKey("pid")) data.setReference((String)item.get("pid"));
if (item.containsKey("key")) data.setSource((String)item.get("key"));
if (this.conventionService.isNullOrEmpty(data.getSource()) && item.containsKey("source")) data.setSource((String) item.get("source"));
if (item.containsKey("reference") && !this.conventionService.isNullOrEmpty(((String)item.get("reference")))){
if (((String)item.get("reference")).contains(":")) {
String[] referenceParts = ((String)item.get("reference")).split(":", 2);
if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(referenceParts[1]);
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource(referenceParts[0]);
}
else if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(((String)item.get("reference")));
}
if (item.containsKey("tag") && !this.conventionService.isNullOrEmpty(((String)item.get("tag")))){
if (((String)item.get("tag")).equalsIgnoreCase("openaire")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire");
} else if (((String)item.get("tag")).equalsIgnoreCase("Open Definition")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("opendefinition");
} else if (((String)item.get("tag")).equalsIgnoreCase("RDA Metadata Schemes")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("rda-metadata-schemes");
} else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE publication repositories 1")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire");
} else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE publication repositories 2")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire");
} else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE publication repositories 3")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire");
} else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE Journals")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire");
} else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE Alternative")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire");
}else if (((String)item.get("tag")).equalsIgnoreCase("Taxonomies")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("taxonomy");
} else if (((String)item.get("tag")).equalsIgnoreCase("OpenAIRE Publications")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("openaire");
} else if (((String)item.get("tag")).equalsIgnoreCase("EOSC Providers")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("eosc");
} else if (((String)item.get("tag")).equalsIgnoreCase("ORCID")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("orcid");
} else if (((String)item.get("tag")).equalsIgnoreCase("Zenodo")) {
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("zenodo");
}
}
if (this.conventionService.isNullOrEmpty(data.getReference()) && item.containsKey("id") && !this.conventionService.isNullOrEmpty(((String)item.get("id")))) {
if (((String)item.get("id")).startsWith("datarepo:re3data/")) {
if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(((String)item.get("id")).substring("datarepo:re3data/".length()));
if (this.conventionService.isNullOrEmpty(data.getSource())) data.setSource("re3data");
}
else if (this.conventionService.isNullOrEmpty(data.getReference())) data.setReference(((String)item.get("id")));
data.setReference((String)item.get("id"));
}
data.setId(UUID.randomUUID());
data.setCreatedAt(Instant.now());
@ -502,7 +711,7 @@ public class DatasetMigrationService {
definitionEntity.setFields(new ArrayList<>());
for (Map.Entry<String, Object> entries : item.entrySet()){
if (entries.getValue() != null && !entries.getKey().equals("name") && !entries.getKey().equals("label")&& !entries.getKey().equals("abbreviation")
&& !entries.getKey().equals("id") && !entries.getKey().equals("pid")&& !entries.getKey().equals("source")) {
&& !entries.getKey().equals("id") && !entries.getKey().equals("key") && !entries.getKey().equals("pid")&& !entries.getKey().equals("source")) {
eu.eudat.commons.types.reference.FieldEntity field = new eu.eudat.commons.types.reference.FieldEntity();
field.setCode(entries.getKey());
field.setDataType(ReferenceFieldDataType.Text);
@ -514,31 +723,260 @@ public class DatasetMigrationService {
if (this.conventionService.isNullOrEmpty(data.getLabel())) data.setLabel(data.getReference());
if (referenceTypeDefinition.getSources().size() == 1) data.setSource(referenceTypeDefinition.getSources().getFirst().getKey());
else {
String key = referenceTypeDefinition.getSources().stream().filter(x-> x.getKey().equalsIgnoreCase((String) item.getOrDefault("source", null))).map(ExternalFetcherBaseSourceConfigurationEntity::getKey).findFirst().orElse(null);
if (this.conventionService.isNullOrEmpty(key)) {
logger.warn("Select first source for reference : " + textValue);
data.setSourceType(ReferenceSourceType.Internal);
if (this.conventionService.isNullOrEmpty(data.getSource()) &&
referenceTypeDefinition.getSources().size() == 1 &&
!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) {
data.setSource(referenceTypeDefinition.getSources().getFirst().getKey());
//throw new MyApplicationException("Could not find reference source for reference : " + textValue); //TODO
} else {
data.setSource(key);
}
if (this.conventionService.isNullOrEmpty(data.getLabel()) || data.getLabel().equals("null")){
logger.warn("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " no label for the reference : " + referenceTypeDataEntity.getReferenceTypeId() + " " + this.jsonHandlingService.toJsonSafe(item)); //TODO
return null;
}
if (this.conventionService.isNullOrEmpty(data.getSource())) throw new MyApplicationException("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " Source not found source for reference : " + referenceTypeDataEntity.getReferenceTypeId() + " " + this.jsonHandlingService.toJsonSafe(item)); //TODO
if (this.conventionService.isNullOrEmpty(data.getReference())) {
data.setReference(this.queryFactory.query(ReferenceQuery.class).sources(data.getSource()).like(data.getLabel()).typeIds(data.getTypeId()).collectAs(new BaseFieldSet().ensure(Reference._reference, Reference._label))
data.setReference(this.queryFactory.query(ReferenceQuery.class).sources(data.getSource()).like(data.getLabel()).typeIds(data.getTypeId()).sourceTypes(ReferenceSourceType.External).collectAs(new BaseFieldSet().ensure(Reference._reference, Reference._label))
.stream().filter(x-> x.getLabel().equals(data.getLabel())).map(ReferenceEntity::getReference).findFirst().orElse(null));//TODO
}
if (this.conventionService.isNullOrEmpty(data.getReference())) {
logger.warn("Local reference created with random id" + textValue);
data.setReference(data.getId().toString().replace("-", ""));
data.setSourceType(ReferenceSourceType.Internal); //TODO
if (referenceTypeDefinition.getSources().stream().filter(x-> x.getType().equals(ReferenceTypeSourceType.API)).map(x-> (ExternalFetcherApiSourceConfigurationEntity)x).anyMatch(x-> x.getUrl().toLowerCase(Locale.ROOT).contains("eestore.paas2.uninett.no"))){
if (data.getLabel().equalsIgnoreCase("Attribution 4.0 International")){
data.setReference("CC-BY-4.0");
} else if (data.getLabel().equalsIgnoreCase("Attribution-NonCommercial 4.0 International")){
data.setReference("CC-BY-NC-4.0");
} else if (data.getLabel().equalsIgnoreCase("Attribution-NonCommercial-NoDerivatives 4.0 International")){
data.setReference("CC-BY-NC-ND-4.0");
} else if (data.getLabel().equalsIgnoreCase("Attribution-NoDerivatives 4.0 International")){
data.setReference("CC-BY-ND-4.0");
} else if (data.getLabel().equalsIgnoreCase("Attribution-NonCommercial-ShareAlike 4.0 International") ){
data.setReference("CC-BY-NC-SA-4.0");
}
} if (data.getTypeId().equals(ReferenceTypeIds.License) && data.getSource().equalsIgnoreCase("OpenAIRE")){
if (data.getLabel().equalsIgnoreCase("Academic Free License 3.0")){
data.setReference("AFL-3.0");
} else if (data.getLabel().equalsIgnoreCase("Attribution Assurance Licenses")){
data.setReference("AFL-3.0");
}
} else if (data.getTypeId().equals(ReferenceTypeIds.Datasets) && data.getSource().equalsIgnoreCase("OpenAIRE")) {
if (data.getLabel().equalsIgnoreCase("Bulk RNA-seq U2OS cells treated with small molecules") && data.getSource().equalsIgnoreCase("OpenAIRE")){
data.setReference("50|_____OmicsDI::7fc1bcd133082463ec1948b82f1ad0ee");
} else if (data.getLabel().equalsIgnoreCase("Genome-wide effect of the microenvironment over the Galectin-7 regulation network in HeLa and SiHa tumors in nude mice.")){
data.setReference("50|_____OmicsDI::4e0de948bf961b96dbdef82797b5a8dc");
}
// else if (data.getLabel().equalsIgnoreCase("other")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("none")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("to be clarified later")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("articles")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Freistilringen")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// }
else if (data.getLabel().equalsIgnoreCase("Randomized Phase 3 Trial of ALIMTA (Pemetrexed) and Carboplatin versus Etoposide and Carboplatin in Extensive-Stage Small Cell Lung Cancer")){
data.setReference("50|datacite____::1df6f72ab95712d5f26ecb43b2d1d303");
}
} else if (data.getTypeId().equals(ReferenceTypeIds.DataRepositories) && data.getSource().equalsIgnoreCase("OpenAIRE")) {
if (data.getLabel().equalsIgnoreCase("INPTDAT")){
data.setReference("re3data_____::r3d100013120");
} if (data.getLabel().equalsIgnoreCase("OpenAPC Global Initiative")){
data.setReference("apc_________::openapc");
} else if (data.getLabel().equalsIgnoreCase("Human Gene and Protein Database")){
data.setReference("fairsharing_::1592");
} else if (data.getLabel().equalsIgnoreCase("Bacterial protein tYrosine Kinase database")){
data.setReference("fairsharing_::1557");
}
// else if (data.getLabel().equalsIgnoreCase("other")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Zenodo")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Zenado")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("none")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("articles")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("https://github.com/sisyphe-re/riot_rpl_udp_scenarios")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else else if (data.getLabel().equalsIgnoreCase("Cool data")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Raw data")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Repository hosted by Zenodo")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("ICPSR - Interuniversity Consortium for Political and Social Research")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("XC7 pharmaceutical development IPF")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("nct")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("UK Data Service")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("figshare")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("University of Strathclyde")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// }
} else if (data.getTypeId().equals(ReferenceTypeIds.Services) && data.getSource().equalsIgnoreCase("eosc")) {
// if (data.getLabel().equalsIgnoreCase("website")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("integration1")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("SeaDataNet Common Data Index (CDI)")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Europeana APIs")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Imaging at Plentzia Marine Station-UPV/EHU")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Spectroscopy")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Library services at Stazione Zoologica Anton Dohrn")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("PRACE Training Portal")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("DARIAH Science Gateway")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Finding Anisotropy Tensor")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Imaging at University of Bergen-SLRC")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Training facilities at Toralla Marine Science Station - Vigo University Marine Research Centre")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Zenodo")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Digital Commons")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// }
}else if (data.getTypeId().equals(ReferenceTypeIds.Registries) && data.getSource().equalsIgnoreCase("rda-metadata-schemes")) {
// if (data.getLabel().equalsIgnoreCase("DataCite Metadata Schema")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("AgMES (Agricultural Metadata Element Set)")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// }
} else if (data.getTypeId().equals(ReferenceTypeIds.Organizations) && data.getSource().equalsIgnoreCase("OpenAIRE")) {
if (data.getLabel().equalsIgnoreCase("Programa de Pós-Graduação em Ciência da Informação Ibict/UFRJ")){
data.setReference("pending_org_::f20d38c291e7d4a7a11d08481eebac44");
} else if (data.getLabel().equalsIgnoreCase("Applied Decision Science (United States)")){
data.setReference("openorgs____::0000093299");
} else if (data.getLabel().equalsIgnoreCase("BIODOL THERAPEUTICS")){
data.setReference("pending_org_::4c550b2917a6fc395bdb79eccdafe950");
} else if (data.getLabel().equalsIgnoreCase("OKKO")){
data.setReference("pending_org_::37ddf98c5fd2d6373fd315b0371fd4ac");
} else if (data.getLabel().equalsIgnoreCase("Health Research Board")){
data.setReference("openorgs____::0000006166");
} else if (data.getLabel().equalsIgnoreCase("Vidzeme University of Applied Sciences")){
data.setReference("openorgs____::0000029719");
}
// else if (data.getLabel().equalsIgnoreCase("Other Lab")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("ATS APPLIED TECH SYSTEMS LTD")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// } else if (data.getLabel().equalsIgnoreCase("Universidade Nova de Lisboa Centro de Linguística")){
// data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
// data.setSourceType(ReferenceSourceType.Internal);
// if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
// }
} else if (data.getTypeId().equals(ReferenceTypeIds.Researcher)){
if (data.getLabel().equalsIgnoreCase("Radosław Cichocki")){
data.setReference("0000-0002-4667-8445");
} else if (data.getLabel().equalsIgnoreCase("Kiranmai Uppuluri")){
data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
data.setSourceType(ReferenceSourceType.Internal);
if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
}
}
}
if (this.conventionService.isNullOrEmpty(data.getReference())) {
data.setReference(data.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
data.setSourceType(ReferenceSourceType.Internal);
if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
logger.warn("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " reference auto created as internal " + referenceTypeDataEntity.getReferenceTypeId() + " for value " + this.jsonHandlingService.toJsonSafe(item));
}
ReferenceEntity existingEntity = referenceMap.getOrDefault(new ReferenceKey(data), null);
if(existingEntity == null){
referenceMap.put(new ReferenceKey(data), data);
if (data.getCreatedById() != null) referenceMap.put(new ReferenceKey(data), data);
this.entityManager.persist(data);
return data;
} else {
@ -547,15 +985,79 @@ public class DatasetMigrationService {
}
private String cleanAsObjectString(String value){
value = value.trim().replace("\\", "");
value = value.trim().replace("[\"{", "[{");
value = value.trim().replace("}\"]", "}]");
value = value.trim().replace("}\", \"{", "}, {");
value = value.trim().replace("}\",\"{", "},{");
private <T> T tryParseJsonAsObjectString(Class<T> type, String value){
T item = this.jsonHandlingService.fromJsonSafe(type, value);
if (item == null) item = this.jsonHandlingService.fromJsonSafe(type, StringEscapeUtils.unescapeJava(value));
if (item == null) item = this.jsonHandlingService.fromJsonSafe(type, StringEscapeUtils.unescapeJson(value));
if (item == null) {
String newValue = StringEscapeUtils.unescapeJava(value);
newValue = newValue.trim().replace("[\"{", "[{");
newValue = newValue.trim().replace("}\"]", "}]");
newValue = newValue.trim().replace("}\", \"{", "}, {");
newValue = newValue.trim().replace("}\",\"{", "},{");
newValue = newValue.trim().replace(", \"{", ", {");
newValue = newValue.trim().replace(",\"{", ",{");
newValue = newValue.trim().replace("}\",", "},");
while (!this.conventionService.isNullOrEmpty(value) && value.startsWith("\"")) value = value.substring(1);
while (!this.conventionService.isNullOrEmpty(value) && value.endsWith("\"")) value = value.substring(0, value.length() - 1);
while (!this.conventionService.isNullOrEmpty(newValue) && newValue.startsWith("\"")) newValue = newValue.substring(1);
while (!this.conventionService.isNullOrEmpty(newValue) && newValue.endsWith("\"")) newValue = newValue.substring(0, newValue.length() - 1);
item = this.jsonHandlingService.fromJsonSafe(type, newValue);
}
if (item == null) {
String newValue = StringEscapeUtils.unescapeJson(value);
newValue = newValue.trim().replace("[\"{", "[{");
newValue = newValue.trim().replace("}\"]", "}]");
newValue = newValue.trim().replace("}\", \"{", "}, {");
newValue = newValue.trim().replace("}\",\"{", "},{");
newValue = newValue.trim().replace(", \"{", ", {");
newValue = newValue.trim().replace(",\"{", ",{");
newValue = newValue.trim().replace("}\",", "},");
while (!this.conventionService.isNullOrEmpty(newValue) && newValue.startsWith("\"")) newValue = newValue.substring(1);
while (!this.conventionService.isNullOrEmpty(newValue) && newValue.endsWith("\"")) newValue = newValue.substring(0, newValue.length() - 1);
item = this.jsonHandlingService.fromJsonSafe(type, newValue);
}
if (item == null) {
String newValue = value.trim().replace("\\", "");
newValue = newValue.trim().replace("[\"{", "[{");
newValue = newValue.trim().replace("}\"]", "}]");
newValue = newValue.trim().replace("}\", \"{", "}, {");
newValue = newValue.trim().replace("}\",\"{", "},{");
newValue = newValue.trim().replace(", \"{", ", {");
newValue = newValue.trim().replace(",\"{", ",{");
newValue = newValue.trim().replace("}\",", "},");
while (!this.conventionService.isNullOrEmpty(newValue) && newValue.startsWith("\"")) newValue = newValue.substring(1);
while (!this.conventionService.isNullOrEmpty(newValue) && newValue.endsWith("\"")) newValue = newValue.substring(0, newValue.length() - 1);
item = this.jsonHandlingService.fromJsonSafe(type, newValue);
}
if (item == null) {
String newValue = value.trim().replace("=", ":");
newValue = newValue.trim().replace("{", "{\"");
newValue = newValue.trim().replace("}", "\"}");
newValue = newValue.trim().replace(":", "\":\"");
newValue = newValue.trim().replace(", ", "\", \"");
newValue = newValue.trim().replace("}\", \"{", "}, {");
newValue = newValue.trim().replace("\"\"", "\"");
newValue = newValue.trim().replace(":\"null,", ":null,");
newValue = newValue.trim().replace(":\"null\",", ":null,");
newValue = newValue.trim().replace(":\"null\"", ":null");
newValue = newValue.trim().replace(":\"null}\"", ":null}");
newValue = newValue.trim().replace("https\":\"", "https:");
newValue = newValue.trim().replace("datarepo\":\"", "datarepo:");
item = this.jsonHandlingService.fromJsonSafe(type, newValue);
}
return item;
}
private String cleanTagString(String value){
value = value.trim().replace("=", ":");
value = value.trim().replace("{", "{\"");
value = value.trim().replace("}", "\"}");
value = value.trim().replace(":", "\":\"");
value = value.trim().replace(", ", "\", \"");
value = value.trim().replace("}\", \"{", "}, {");
return value;
}

4
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java
View File

@ -82,14 +82,14 @@ public class DescriptionTemplateXmlMigrationService {
Document document = XmlBuilder.fromXml(item.getDefinition());
if (document == null){
logger.error("Migrate DescriptionTemplate " + item.getId() + " failed read xml");
continue;
throw new MyApplicationException("Migrate DescriptionTemplate " + item.getId() + " failed read xml");
}
ViewStyleModel viewStyleModel = null;
try {
viewStyleModel = new ViewStyleModel().fromXml(document.getDocumentElement());
} catch (Exception ex){
logger.error("Migrate DescriptionTemplate " + item.getId() + " failed read xml");
continue;
throw new MyApplicationException("Migrate DescriptionTemplate " + item.getId() + " failed read xml");
}
item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(viewStyleModel, referenceTypeEntityMap)));
this.entityManager.merge(item);

4
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java
View File

@ -78,23 +78,19 @@ public class DmpDatasetProfileMigrationService {
DmpDatasetProfileData profileData = jsonHandlingService.fromJson(DmpDatasetProfileData.class, item.getData());
if (profileData == null || profileData.dmpSectionIndex == null || profileData.dmpSectionIndex.isEmpty()){
logger.error("Migrate DmpDatasetProfile " + item.getId() + " failed no section info ");
throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " failed no section info ");
}
DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null);
if (definition == null){
logger.error("Migrate DmpDatasetProfile " + item.getId() + " failed blueprint definition not found for blueprint " + item.getDmp().getProfile().getId());
throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " failed blueprint definition not found for blueprint " + item.getDmp().getProfile().getId());
}
for (int sectionIndex: profileData.dmpSectionIndex) {
if (definition.getSections() == null || definition.getSections().size() <= sectionIndex) {
logger.error("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex);
throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex);
}
UUID sectionId = definition.getSections().get(sectionIndex).getId();
if (sectionId == null) {
logger.error("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex);
throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex);
}

13
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java
View File

@ -172,7 +172,6 @@ public class DmpMigrationService {
data.setLanguage("en");
}
if (data.getCreatorId() == null){
logger.warn("Migration skipped creator not found " + item.getId());
throw new MyApplicationException("Migration skipped creator not found " + item.getId());
}
@ -196,12 +195,10 @@ public class DmpMigrationService {
if (organisations == null || organisations.isEmpty()) return;
if (referenceTypeFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Organizations)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
logger.warn("Migration failed Organizations field not found " + item.getId());
throw new MyApplicationException("Migration failed Organizations field not found " + item.getId());
}
@ -222,12 +219,10 @@ public class DmpMigrationService {
Project project = item.getProject();
if (project == null) return;
if (referenceTypeFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Project)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
logger.warn("Migration failed Project field not found " + item.getId());
throw new MyApplicationException("Migration failed Project field not found " + item.getId());
}
@ -246,12 +241,10 @@ public class DmpMigrationService {
Set<Researcher> researchers = item.getResearchers();
if (researchers == null || researchers.isEmpty()) return;
if (referenceTypeFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Researcher)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
logger.warn("Migration failed Researchers field not found " + item.getId());
throw new MyApplicationException("Migration failed Researchers field not found " + item.getId());
}
@ -273,12 +266,10 @@ public class DmpMigrationService {
if (grant == null) return;
if (referenceTypeFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Grants)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
logger.warn("Migration failed Grant field not found " + item.getId());
throw new MyApplicationException("Migration failed Grant field not found " + item.getId());
}
@ -294,7 +285,6 @@ public class DmpMigrationService {
if (grant.getFunder() != null){
ReferenceTypeFieldEntity funderField = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Funder)).findFirst().orElse(null);
if (funderField == null) {
logger.warn("Migration failed Funder field not found " + item.getId());
throw new MyApplicationException("Migration failed Funder field not found " + item.getId());
}
DmpReferenceEntity founder = new DmpReferenceEntity();
@ -313,12 +303,10 @@ public class DmpMigrationService {
if (model.getExtraProperties() != null) {
if (model.getExtraProperties().containsKey("license") && model.getExtraProperties().get("license") != null) {
if (referenceTypeFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.License)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
logger.warn("Migration failed License field not found " + item.getId());
throw new MyApplicationException("Migration failed License field not found " + item.getId());
}
@ -388,7 +376,6 @@ public class DmpMigrationService {
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
licenseIdByName.put(licensePid, referenceEntity.getId());
logger.debug("License '{}' migrated", licensePid);
}
DmpReferenceEntity dmpReferenceEntity = new DmpReferenceEntity();

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ExternalDatasetMigrationService.java
View File

@ -42,7 +42,7 @@ public class ExternalDatasetMigrationService {
entityManager.detach(item);
if (item.getReference() == null || !item.getReference().contains(":")){
logger.warn("Reference generated because is null ExternalDataset " + item.getId());
item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT));
item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
}
eu.old.eudat.models.data.externaldataset.ExternalDatasetModel model = new eu.old.eudat.models.data.externaldataset.ExternalDatasetModel().fromDataModel(item);

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/FunderMigrationService.java
View File

@ -46,7 +46,7 @@ public class FunderMigrationService {
entityManager.detach(item);
if (item.getReference() == null || !item.getReference().contains(":")){
logger.warn("Reference generated because is null Funder " + item.getId());
item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT));
item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
}
eu.old.eudat.models.data.funder.Funder model = new eu.old.eudat.models.data.funder.Funder().fromDataModel(item);

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/GrantMigrationService.java
View File

@ -52,7 +52,7 @@ public class GrantMigrationService {
entityManager.detach(item);
if (item.getReference() == null || !item.getReference().contains(":")){
logger.warn("Reference generated because is null Grant " + item.getId());
item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT));
item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
}
String[] referenceParts = item.getReference().split(":", 2);

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/OrganizationMigrationService.java
View File

@ -51,7 +51,7 @@ public class OrganizationMigrationService {
entityManager.detach(item);
if (item.getReference() == null || !item.getReference().contains(":")){
logger.warn("Reference generated because is null Organisation " + item.getId());
item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT));
item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
}
eu.old.eudat.models.data.dmp.Organisation model = new eu.old.eudat.models.data.dmp.Organisation().fromDataModel(item);

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ProjectMigrationService.java
View File

@ -52,7 +52,7 @@ public class ProjectMigrationService {
entityManager.detach(item);
if (item.getReference() == null || !item.getReference().contains(":")){
logger.warn("Reference generated because is null Project " + item.getId());
item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT));
item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
}
eu.old.eudat.models.data.project.Project model = new eu.old.eudat.models.data.project.Project().fromDataModel(item);

4
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ReferenceTypeIds.java
View File

@ -1,5 +1,6 @@
package eu.old.eudat.migration;
import java.util.List;
import java.util.UUID;
public class ReferenceTypeIds {
@ -18,4 +19,7 @@ public class ReferenceTypeIds {
public static UUID Journal = UUID.fromString("8ec7556b-749d-4c4a-a4b9-43d064693795");
public static UUID PubRepositories = UUID.fromString("1e927daa-b856-443f-96da-22f325f7322f");
public static List<UUID> KnownReferenceTypeIds = List.of(DataRepositories, Datasets, Funder, Grants, Organizations,
Project, Registries, Researcher, Services, License, Taxonomy, Publication, Journal, PubRepositories);
}

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/RegistryMigrationService.java
View File

@ -51,7 +51,7 @@ public class RegistryMigrationService {
entityManager.detach(item);
if (item.getReference() == null || !item.getReference().contains(":")){
logger.warn("Reference generated because is null Registry " + item.getId());
item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT));
item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
}
eu.old.eudat.models.data.registries.RegistryModel model = new eu.old.eudat.models.data.registries.RegistryModel().fromDataModel(item);

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ResearcherMigrationService.java
View File

@ -52,7 +52,7 @@ public class ResearcherMigrationService {
entityManager.detach(item);
if (item.getReference() == null || !item.getReference().contains(":")){
logger.warn("Reference generated because is null Researcher " + item.getId());
item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT));
item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
}
eu.old.eudat.models.data.dmp.Researcher model = new eu.old.eudat.models.data.dmp.Researcher().fromDataModel(item);

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ServiceMigrationService.java
View File

@ -50,7 +50,7 @@ public class ServiceMigrationService {
entityManager.detach(item);
if (item.getReference() == null || !item.getReference().contains(":")){
logger.warn("Reference generated because is null Service " + item.getId());
item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT));
item.setReference(InternalReferenceSource + ":" + item.getId().toString().replace("-", "").toLowerCase(Locale.ROOT));
}
eu.old.eudat.models.data.services.ServiceModel model = new eu.old.eudat.models.data.services.ServiceModel().fromDataModel(item);

3
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java
View File

@ -51,14 +51,12 @@ public class TagMigrationService {
Set<String> savedTagNames = new HashSet<>();
Map<String, UUID> savedTagIdsByName = new HashMap<>();
DatasetCriteria criteria = new DatasetCriteria();
List<Dataset> items;
do {
items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dataset tags " + page * PageSize + " of " + total);
criteria.setIds(items.stream().map(Dataset::getId).toList());
List<eu.old.eudat.elastic.entities.Dataset> elasticDatasets = this.datasetRepository.findByIds(items.stream().map(x -> x.getId().toString()).toList());
for (Dataset item : items) {
@ -83,7 +81,6 @@ public class TagMigrationService {
tagEntity.setIsActive(IsActive.Active);
savedTagIdsByName.put(tag.getName(), tagEntity.getId());
this.entityManager.persist(tagEntity);
logger.debug("New tag saved '{}'", tag.getName());
}
DescriptionTagEntity descriptionTagEntity = new DescriptionTagEntity();
descriptionTagEntity.setId(UUID.randomUUID());

354
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -90,6 +90,7 @@ public class MigrationController {
@GetMapping("all")
@Transactional
public boolean migrateAll() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException, NoSuchFieldException, InvalidApplicationException, TransformerException, URISyntaxException {
// //Reference
// this.dataRepositoryMigrationService.migrate();
// this.externalDatasetMigrationService.migrate();
// this.funderMigrationService.migrate();
@ -100,6 +101,7 @@ public class MigrationController {
// this.researcherMigrationService.migrate();
// this.serviceMigrationService.migrate();
//
// //User
// this.userContactInfoMigrationService.migrate();
// this.userMigrationService.migrate();
//
@ -107,188 +109,190 @@ public class MigrationController {
// this.dmpBlueprintXmlMigrationService.migrate();
// this.descriptionTemplateXmlMigrationService.migrate();
//
// //Dmp
// this.dmpMigrationService.migrate();
// this.dmpDatasetProfileMigrationService.migrate();
this.datasetMigrationService.migrate(); //TODO
// this.tagMigrationService.migrate();
//
// this.dmpUserMigrationService.migrate();
//
//Description
this.datasetMigrationService.migrate(); //TODO
// this.referenceMigrationService.migrateDatasetReferences();
//
// this.tagMigrationService.migrate();
// this.storageFileMigrationService.migrate();
return true;
throw new RuntimeException("");
//return true;
}
@GetMapping("dmp-blueprints-xml")
@Transactional
public boolean migrateDmpBlueprint() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException {
this.dmpBlueprintXmlMigrationService.migrate();
return true;
}
@GetMapping("storage-files")
@Transactional
public boolean migrateStorageFile() throws IOException {
this.storageFileMigrationService.migrate();
return true;
}
@GetMapping("description-template-xml")
@Transactional
public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
this.descriptionTemplateXmlMigrationService.migrate();
return true;
}
@GetMapping("description-template-xml-clean-invalid-reference-types")
@Transactional
public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate();
return true;
}
@GetMapping("contacts")
@Transactional
public boolean migrateContacts() {
this.userContactInfoMigrationService.migrate();
return true;
}
@GetMapping("users")
@Transactional
public boolean migrateUsers() throws JsonProcessingException {
this.userMigrationService.migrate();
return true;
}
@GetMapping("dmps")
@Transactional
public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException {
this.dmpMigrationService.migrate();
return true;
}
@GetMapping("datasets")
@Transactional
public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
this.datasetMigrationService.migrate();
return true;
}
@GetMapping("tags")
@Transactional
public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
this.tagMigrationService.migrate();
return true;
}
@GetMapping("dmp-dataset-profiles")
@Transactional
public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
this.dmpDatasetProfileMigrationService.migrate();
return true;
}
@GetMapping("references")
@Transactional
public boolean migrateReferences() {
this.dataRepositoryMigrationService.migrate();
this.externalDatasetMigrationService.migrate();
this.funderMigrationService.migrate();
this.grantMigrationService.migrate();
this.organizationMigrationService.migrate();
this.projectMigrationService.migrate();
this.registryMigrationService.migrate();
this.researcherMigrationService.migrate();
this.serviceMigrationService.migrate();
return true;
}
@GetMapping("dmp-users")
@Transactional
public boolean migrateDmpUsers() {
this.dmpUserMigrationService.migrate();
return true;
}
@GetMapping("dataset-references")
@Transactional
public boolean migrateDatasetReferences() {
this.referenceMigrationService.migrateDatasetReferences();
return true;
}
@GetMapping("organizations")
@Transactional
public boolean migrateOrganizations() {
this.organizationMigrationService.migrate();
return true;
}
@GetMapping("dataRepositories")
@Transactional
public boolean migrateDataRepositories() {
this.dataRepositoryMigrationService.migrate();
return true;
}
@GetMapping("externalDatasets")
@Transactional
public boolean migrateExternalDatasets() {
this.externalDatasetMigrationService.migrate();
return true;
}
@GetMapping("funders")
@Transactional
public boolean migrateFunders() {
this.funderMigrationService.migrate();
return true;
}
@GetMapping("grants")
@Transactional
public boolean migrateGrants() {
this.grantMigrationService.migrate();
return true;
}
@GetMapping("projects")
@Transactional
public boolean migrateProjects() {
this.projectMigrationService.migrate();
return true;
}
@GetMapping("registries")
@Transactional
public boolean migrateRegistries() {
this.registryMigrationService.migrate();
return true;
}
@GetMapping("researchers")
@Transactional
public boolean migrateResearchers() {
this.researcherMigrationService.migrate();
return true;
}
@GetMapping("services")
@Transactional
public boolean migrateServices() {
this.serviceMigrationService.migrate();
return true;
}
// @GetMapping("dmp-blueprints-xml")
// @Transactional
// public boolean migrateDmpBlueprint() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException {
// this.dmpBlueprintXmlMigrationService.migrate();
// return true;
//
// }
//
// @GetMapping("storage-files")
// @Transactional
// public boolean migrateStorageFile() throws IOException {
// this.storageFileMigrationService.migrate();
// return true;
//
// }
//
// @GetMapping("description-template-xml")
// @Transactional
// public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
// this.descriptionTemplateXmlMigrationService.migrate();
// return true;
//
// }
//
// @GetMapping("description-template-xml-clean-invalid-reference-types")
// @Transactional
// public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
// this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate();
// return true;
//
// }
//
// @GetMapping("contacts")
// @Transactional
// public boolean migrateContacts() {
// this.userContactInfoMigrationService.migrate();
// return true;
//
// }
//
// @GetMapping("users")
// @Transactional
// public boolean migrateUsers() throws JsonProcessingException {
// this.userMigrationService.migrate();
// return true;
//
// }
//
//
// @GetMapping("dmps")
// @Transactional
// public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException {
// this.dmpMigrationService.migrate();
// return true;
// }
//
// @GetMapping("datasets")
// @Transactional
// public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
// this.datasetMigrationService.migrate();
// return true;
// }
//
// @GetMapping("tags")
// @Transactional
// public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
// this.tagMigrationService.migrate();
// return true;
// }
//
// @GetMapping("dmp-dataset-profiles")
// @Transactional
// public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
// this.dmpDatasetProfileMigrationService.migrate();
// return true;
// }
//
// @GetMapping("references")
// @Transactional
// public boolean migrateReferences() {
// this.dataRepositoryMigrationService.migrate();
// this.externalDatasetMigrationService.migrate();
// this.funderMigrationService.migrate();
// this.grantMigrationService.migrate();
// this.organizationMigrationService.migrate();
// this.projectMigrationService.migrate();
// this.registryMigrationService.migrate();
// this.researcherMigrationService.migrate();
// this.serviceMigrationService.migrate();
// return true;
// }
//
// @GetMapping("dmp-users")
// @Transactional
// public boolean migrateDmpUsers() {
// this.dmpUserMigrationService.migrate();
// return true;
// }
//
// @GetMapping("dataset-references")
// @Transactional
// public boolean migrateDatasetReferences() {
// this.referenceMigrationService.migrateDatasetReferences();
// return true;
// }
//
// @GetMapping("organizations")
// @Transactional
// public boolean migrateOrganizations() {
// this.organizationMigrationService.migrate();
// return true;
// }
//
//
// @GetMapping("dataRepositories")
// @Transactional
// public boolean migrateDataRepositories() {
// this.dataRepositoryMigrationService.migrate();
// return true;
// }
//
// @GetMapping("externalDatasets")
// @Transactional
// public boolean migrateExternalDatasets() {
// this.externalDatasetMigrationService.migrate();
// return true;
// }
//
// @GetMapping("funders")
// @Transactional
// public boolean migrateFunders() {
// this.funderMigrationService.migrate();
// return true;
// }
//
// @GetMapping("grants")
// @Transactional
// public boolean migrateGrants() {
// this.grantMigrationService.migrate();
// return true;
// }
//
// @GetMapping("projects")
// @Transactional
// public boolean migrateProjects() {
// this.projectMigrationService.migrate();
// return true;
// }
//
// @GetMapping("registries")
// @Transactional
// public boolean migrateRegistries() {
// this.registryMigrationService.migrate();
// return true;
// }
//
// @GetMapping("researchers")
// @Transactional
// public boolean migrateResearchers() {
// this.researcherMigrationService.migrate();
// return true;
// }
//
// @GetMapping("services")
// @Transactional
// public boolean migrateServices() {
// this.serviceMigrationService.migrate();
// return true;
// }
}