MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

migration fixes

This commit is contained in:
Efstratios Giannopoulos 2024-04-25 11:35:08 +03:00
parent c27e98acd8
commit 33d9bab4bf
3 changed files with 103 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -489,6 +489,16 @@ public class DatasetMigrationService {
ref.put("source", "opendefinition");
} else if (ReferenceTypeIds.PubRepositories.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "opendefinition");
} else if (ReferenceTypeIds.Grants.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "openaire");
} else if (ReferenceTypeIds.Project.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "project");
} else if (ReferenceTypeIds.Taxonomy.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "taxonomy");
} else if (ReferenceTypeIds.Publication.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "openaire");
} else if (ReferenceTypeIds.Currency.equals(referenceTypeDataEntity.getReferenceTypeId())) {
ref.put("source", "currencies");
} else if (ReferenceTypeIds.Researcher.equals(referenceTypeDataEntity.getReferenceTypeId())) {
Matcher pattern = Pattern.compile("\\((.*?)\\)").matcher(textValue);
if (pattern.find()) {
@ -504,40 +514,37 @@ public class DatasetMigrationService {
}
if (ref.containsKey("source")) references = List.of(ref).toArray(new Map[1]);
} else {
String source = null;
if (!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) {
ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null);
if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
//if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
source = referenceTypeDefinition.getSources().getFirst().getKey();
String[] parts = null;
if (textValue.startsWith("[\"")){
parts = migrationTools.tryParseJsonAsObjectString(String[].class, textValue);
} else{
if (textValue.startsWith("[")) textValue = textValue.substring(1).trim();
if (textValue.endsWith("]")) textValue = textValue.substring(0, textValue.length() -1).trim();
parts = textValue.split(",");
ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null);
if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
//if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
String source = referenceTypeDefinition.getSources().getFirst().getKey();
String[] parts = null;
if (textValue.startsWith("[\"")){
parts = migrationTools.tryParseJsonAsObjectString(String[].class, textValue);
} else{
if (textValue.startsWith("[")) textValue = textValue.substring(1).trim();
if (textValue.endsWith("]")) textValue = textValue.substring(0, textValue.length() -1).trim();
parts = textValue.split(",");
}
if (parts != null) {
List<Map<String, Object>> items = new ArrayList<>();
for (String part : parts) {
if (this.conventionService.isNullOrEmpty(part)) continue;
part = part.trim().replace(",", "").trim();
if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue;
Map<String, Object> ref = new HashMap<>();
ref.put("name", part);
ref.put("source", source);
items.add(ref);
}
if (parts != null) {
List<Map<String, Object>> items = new ArrayList<>();
for (String part : parts) {
if (this.conventionService.isNullOrEmpty(part)) continue;
part = part.trim().replace(",", "").trim();
if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue;
Map<String, Object> ref = new HashMap<>();
ref.put("name", part);
ref.put("source", source);
items.add(ref);
}
if (items.isEmpty()) break;
references = items.toArray(new Map[items.size()]);
}
}
if (items.isEmpty()) break;
references = items.toArray(new Map[items.size()]);
}
}
}
if (references == null && !ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())){
if (references == null){
if (textValue.equalsIgnoreCase("[\"dublin core\",\"{\\\"id\\\":\\\"metadataschema:rd-alliance/dublin-core\\\",\\\"label\\\":\\\"Dublin Core\\\",\\\"source\\\":\\\"\\\",\\\"uri\\\":\\\"http://dublincore.org\\\"}\"]")){
Map<String, Object> ref = new HashMap<>();
@ -733,8 +740,7 @@ public class DatasetMigrationService {
if (this.conventionService.isNullOrEmpty(data.getLabel())) data.setLabel(data.getReference());
if (this.conventionService.isNullOrEmpty(data.getSource()) &&
referenceTypeDefinition.getSources().size() == 1 &&
!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) {
referenceTypeDefinition.getSources().size() == 1) {
data.setSource(referenceTypeDefinition.getSources().getFirst().getKey());
}
if (this.conventionService.isNullOrEmpty(data.getLabel()) || data.getLabel().equals("null")){
@ -742,7 +748,7 @@ public class DatasetMigrationService {
return null;
}
if (this.conventionService.isNullOrEmpty(data.getSource())) throw new MyApplicationException("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " Source not found source for reference : " + referenceTypeDataEntity.getReferenceTypeId() + " " + this.jsonHandlingService.toJsonSafe(item)); //TODO
if (this.conventionService.isNullOrEmpty(data.getSource())) throw new MyApplicationException("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " Source not found source for reference : " + referenceTypeDataEntity.getReferenceTypeId() + " " + this.jsonHandlingService.toJsonSafe(item)); //TODO
if (this.conventionService.isNullOrEmpty(data.getReference())) {
data.setReference(this.queryFactory.query(ReferenceQuery.class).sources(data.getSource()).like(data.getLabel()).typeIds(data.getTypeId()).sourceTypes(ReferenceSourceType.External).collectAs(new BaseFieldSet().ensure(Reference._reference, Reference._label))

58
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java
View File

@ -14,6 +14,7 @@ import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.data.converters.enums.DatabaseEnum;
import eu.eudat.model.DescriptionTemplate;
import eu.eudat.query.DescriptionTemplateQuery;
import eu.eudat.query.ReferenceTypeQuery;
import eu.old.eudat.logic.services.helpers.HelpersService;
import eu.old.eudat.logic.utilities.builders.XmlBuilder;
import eu.old.eudat.models.data.components.commons.Multiplicity;
@ -28,6 +29,7 @@ import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import jakarta.ws.rs.NotSupportedException;
import jakarta.xml.bind.JAXBException;
import org.elasticsearch.common.inject.Guice;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
@ -75,6 +77,7 @@ public class DescriptionTemplateXmlMigrationService {
int page = 0;
Map<String, ReferenceTypeEntity> referenceTypeEntityMap = new HashMap<>();
Map<UUID, ReferenceTypeEntity> knownReferenceEntities = this.queryFactory.query(ReferenceTypeQuery.class).ids(ReferenceTypeIds.KnownReferenceTypeIds).collect().stream().collect(Collectors.toMap(ReferenceTypeEntity::getId, x-> x));
List<DescriptionTemplateEntity> items;
do {
DescriptionTemplateQuery query = this.queryFactory.query(DescriptionTemplateQuery.class);
@ -99,7 +102,7 @@ public class DescriptionTemplateXmlMigrationService {
logger.error("Migrate DescriptionTemplate " + item.getId() + " failed read xml");
throw new MyApplicationException("Migrate DescriptionTemplate " + item.getId() + " failed read xml");
}
item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(viewStyleModel, referenceTypeEntityMap)));
item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(viewStyleModel, referenceTypeEntityMap, knownReferenceEntities)));
this.entityManager.merge(item);
}
@ -115,7 +118,7 @@ public class DescriptionTemplateXmlMigrationService {
this.entityManager.flush();
}
private @NotNull DefinitionEntity buildDefinitionEntity(ViewStyleModel persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
private @NotNull DefinitionEntity buildDefinitionEntity(ViewStyleModel persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
DefinitionEntity data = new DefinitionEntity();
if (persist == null)
return data;
@ -123,14 +126,14 @@ public class DescriptionTemplateXmlMigrationService {
Map<String, List<Section>> sectionPerPage = mapSectionsToPage(persist);
data.setPages(new ArrayList<>());
for (Page pagePersist : persist.getPages()) {
data.getPages().add(this.buildPageEntity(pagePersist, sectionPerPage.get(pagePersist.getId()), referenceTypeEntityMap));
data.getPages().add(this.buildPageEntity(pagePersist, sectionPerPage.get(pagePersist.getId()), referenceTypeEntityMap, knownReferenceEntities));
}
}
return data;
}
private @NotNull SectionEntity buildSectionEntity(Section persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
private @NotNull SectionEntity buildSectionEntity(Section persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
SectionEntity data = new SectionEntity();
if (persist == null)
return data;
@ -146,21 +149,21 @@ public class DescriptionTemplateXmlMigrationService {
if (!this.conventionService.isListNullOrEmpty(persist.getSections())) {
data.setSections(new ArrayList<>());
for (Section sectionPersist : persist.getSections()) {
data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap));
data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap, knownReferenceEntities));
}
}
if (!this.conventionService.isListNullOrEmpty(persist.getFieldSets())) {
data.setFieldSets(new ArrayList<>());
for (FieldSet fieldSetPersist : persist.getFieldSets()) {
data.getFieldSets().add(this.buildFieldSetEntity(fieldSetPersist, referenceTypeEntityMap));
data.getFieldSets().add(this.buildFieldSetEntity(fieldSetPersist, referenceTypeEntityMap, knownReferenceEntities));
}
}
return data;
}
private @NotNull FieldSetEntity buildFieldSetEntity(FieldSet persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
private @NotNull FieldSetEntity buildFieldSetEntity(FieldSet persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
FieldSetEntity data = new FieldSetEntity();
if (persist == null)
return data;
@ -180,13 +183,13 @@ public class DescriptionTemplateXmlMigrationService {
if (!this.conventionService.isListNullOrEmpty(persist.getFields())) {
data.setFields(new ArrayList<>());
for (Field fieldPersist : persist.getFields()) {
data.getFields().add(this.buildFieldEntity(fieldPersist, referenceTypeEntityMap));
data.getFields().add(this.buildFieldEntity(fieldPersist, referenceTypeEntityMap, knownReferenceEntities));
}
}
return data;
}
private @NotNull FieldEntity buildFieldEntity(Field persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
private @NotNull FieldEntity buildFieldEntity(Field persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
FieldEntity data = new FieldEntity();
if (persist == null)
return data;
@ -213,7 +216,7 @@ public class DescriptionTemplateXmlMigrationService {
}).toList());
data.setIncludeInExport(persist.getExport());
if (persist.getData() != null) {
data.setData(this.buildFieldDataEntity(persist.getData(), persist.getViewStyle().getRenderStyle(), referenceTypeEntityMap));
data.setData(this.buildFieldDataEntity(persist.getData(), persist.getViewStyle().getRenderStyle(), referenceTypeEntityMap, knownReferenceEntities));
if (persist.getViewStyle().getRenderStyle().equals("externalDatasets")){
FieldDataExternalDatasetType type = null;
try {
@ -239,7 +242,7 @@ public class DescriptionTemplateXmlMigrationService {
return data;
}
private BaseFieldDataEntity buildFieldDataEntity(FieldData<?> persist, String renderStyle, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
private BaseFieldDataEntity buildFieldDataEntity(FieldData<?> persist, String renderStyle, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
if (renderStyle.equals("checkBox")){
return this.buildLabelDataEntity(persist.getLabel(), FieldType.CHECK_BOX);
}
@ -369,7 +372,7 @@ public class DescriptionTemplateXmlMigrationService {
return data;
}
case "autocomplete":{
ReferenceTypeEntity referenceType = this.resolveReferenceTypeEntityAutoCompleteData(((AutoCompleteData)persist), referenceTypeEntityMap);
ReferenceTypeEntity referenceType = this.resolveReferenceTypeEntityAutoCompleteData(((AutoCompleteData)persist), referenceTypeEntityMap, knownReferenceEntities);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((AutoCompleteData)persist).getMultiAutoComplete(), referenceType.getId());
}
}
@ -461,7 +464,7 @@ public class DescriptionTemplateXmlMigrationService {
private @NotNull PageEntity buildPageEntity(Page persist, List<Section> sections, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
private @NotNull PageEntity buildPageEntity(Page persist, List<Section> sections, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
PageEntity data = new PageEntity();
if (persist == null)
return data;
@ -473,7 +476,7 @@ public class DescriptionTemplateXmlMigrationService {
if (!this.conventionService.isListNullOrEmpty(sections)) {
data.setSections(new ArrayList<>());
for (Section sectionPersist : sections) {
data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap));
data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap, knownReferenceEntities));
}
}
return data;
@ -511,10 +514,13 @@ public class DescriptionTemplateXmlMigrationService {
}
}
public ReferenceTypeEntity resolveReferenceTypeEntityAutoCompleteData(AutoCompleteData persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
public ReferenceTypeEntity resolveReferenceTypeEntityAutoCompleteData(AutoCompleteData persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
ReferenceTypeDefinitionEntity definitionEntity = this.buildReferenceTypeDefinitionEntity(persist);
String hash = this.referenceTypeCalculatedHash(definitionEntity);
ReferenceTypeEntity data = referenceTypeEntityMap.getOrDefault(hash, null);
ReferenceTypeEntity data = this.getReferenceTypeEntityByUrl(definitionEntity, knownReferenceEntities);
if (data == null){
data = referenceTypeEntityMap.getOrDefault(hash, null);
}
if (data == null) {
data = new ReferenceTypeEntity();
data.setId(UUID.randomUUID());
@ -528,6 +534,26 @@ public class DescriptionTemplateXmlMigrationService {
}
return data;
}
private ReferenceTypeEntity getReferenceTypeEntityByUrl(ReferenceTypeDefinitionEntity definitionEntity, Map<UUID, ReferenceTypeEntity> knownReferenceEntities){
if (definitionEntity == null || this.conventionService.isListNullOrEmpty(definitionEntity.getSources()) || definitionEntity.getSources().size() != 1) return null;
ExternalFetcherApiSourceConfigurationEntity externalFetcherApiSourceConfigurationEntity = (ExternalFetcherApiSourceConfigurationEntity)definitionEntity.getSources().getFirst();
if (externalFetcherApiSourceConfigurationEntity == null || this.conventionService.isNullOrEmpty(externalFetcherApiSourceConfigurationEntity.getUrl())) return null;
if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/license/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.License);
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://zenodo.org/api/licenses".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.License);
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://zenodo.org/api/licenses?q={like}".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.License);
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("http://api.openaire.eu/search/publications".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Publication);
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("http://api.openaire.eu/search/publications?format=json&title={like}".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Publication);
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://services.openaire.eu/openaire/ds/searchdetails/{page}/{pageSize}?requestSortBy=id&order=ASCENDING".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Publication);
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/datarepo/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.DataRepositories);
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://services.openaire.eu/search/v2/api/datasources?q={like}&fq=datasourcetypeuiname".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.DataRepositories);
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://services.openaire.eu/search/v2/api/resources?query=oaftype exact datasource and {like} and ( datasourcetypeuiid = \"pubsrepository::journal\" or datasourcetypeuiid = \"aggregator::pubsrepository::journals\")&page=0&size=60&format=json".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Journal);
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/metadataschema/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Registries);
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/taxonomy/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Taxonomy);
return null;
}
public String referenceTypeCalculatedHash(ReferenceTypeDefinitionEntity definitionEntity) {
List<String> sourceKeys = new ArrayList<>();

44
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -90,29 +90,29 @@ public class MigrationController {
@GetMapping("all")
@Transactional
public boolean migrateAll() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException, NoSuchFieldException, InvalidApplicationException, TransformerException, URISyntaxException {
//Reference
this.dataRepositoryMigrationService.migrate();
this.externalDatasetMigrationService.migrate();
this.funderMigrationService.migrate();
this.grantMigrationService.migrate();
this.organizationMigrationService.migrate();
this.projectMigrationService.migrate();
this.registryMigrationService.migrate();
this.researcherMigrationService.migrate();
this.serviceMigrationService.migrate();
// //Reference
// this.dataRepositoryMigrationService.migrate();
// this.externalDatasetMigrationService.migrate();
// this.funderMigrationService.migrate();
// this.grantMigrationService.migrate();
// this.organizationMigrationService.migrate();
// this.projectMigrationService.migrate();
// this.registryMigrationService.migrate();
// this.researcherMigrationService.migrate();
// this.serviceMigrationService.migrate();
//
// //User
// this.userContactInfoMigrationService.migrate();
// this.userMigrationService.migrate();
//
// //XML recreate
// this.dmpBlueprintXmlMigrationService.migrate();
// this.descriptionTemplateXmlMigrationService.migrate();
//User
this.userContactInfoMigrationService.migrate();
this.userMigrationService.migrate();
//XML recreate
this.dmpBlueprintXmlMigrationService.migrate();
this.descriptionTemplateXmlMigrationService.migrate();
//Dmp
this.dmpMigrationService.migrate();
this.dmpDatasetProfileMigrationService.migrate();
this.dmpUserMigrationService.migrate();
// //Dmp
// this.dmpMigrationService.migrate();
// this.dmpDatasetProfileMigrationService.migrate();
// this.dmpUserMigrationService.migrate();
//Description
this.datasetMigrationService.migrate();