From 33d9bab4bf362f700532fdcc664ccad722b57374 Mon Sep 17 00:00:00 2001 From: sgiannopoulos Date: Thu, 25 Apr 2024 11:35:08 +0300 Subject: [PATCH] migration fixes --- .../migration/DatasetMigrationService.java | 72 ++++++++++--------- ...escriptionTemplateXmlMigrationService.java | 58 ++++++++++----- .../migration/MigrationController.java | 44 ++++++------ 3 files changed, 103 insertions(+), 71 deletions(-) diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java index 3092b9ce1..38a7e0894 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java @@ -489,6 +489,16 @@ public class DatasetMigrationService { ref.put("source", "opendefinition"); } else if (ReferenceTypeIds.PubRepositories.equals(referenceTypeDataEntity.getReferenceTypeId())) { ref.put("source", "opendefinition"); + } else if (ReferenceTypeIds.Grants.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "openaire"); + } else if (ReferenceTypeIds.Project.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "project"); + } else if (ReferenceTypeIds.Taxonomy.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "taxonomy"); + } else if (ReferenceTypeIds.Publication.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "openaire"); + } else if (ReferenceTypeIds.Currency.equals(referenceTypeDataEntity.getReferenceTypeId())) { + ref.put("source", "currencies"); } else if (ReferenceTypeIds.Researcher.equals(referenceTypeDataEntity.getReferenceTypeId())) { Matcher pattern = Pattern.compile("\\((.*?)\\)").matcher(textValue); if (pattern.find()) { @@ -504,40 +514,37 @@ public class DatasetMigrationService { } if (ref.containsKey("source")) references = List.of(ref).toArray(new Map[1]); } else { - String source = null; - if (!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) { - ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null); - if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); - if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); - //if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); - source = referenceTypeDefinition.getSources().getFirst().getKey(); - String[] parts = null; - if (textValue.startsWith("[\"")){ - parts = migrationTools.tryParseJsonAsObjectString(String[].class, textValue); - } else{ - if (textValue.startsWith("[")) textValue = textValue.substring(1).trim(); - if (textValue.endsWith("]")) textValue = textValue.substring(0, textValue.length() -1).trim(); - parts = textValue.split(","); + ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null); + if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + //if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue)); + String source = referenceTypeDefinition.getSources().getFirst().getKey(); + String[] parts = null; + if (textValue.startsWith("[\"")){ + parts = migrationTools.tryParseJsonAsObjectString(String[].class, textValue); + } else{ + if (textValue.startsWith("[")) textValue = textValue.substring(1).trim(); + if (textValue.endsWith("]")) textValue = textValue.substring(0, textValue.length() -1).trim(); + parts = textValue.split(","); + } + if (parts != null) { + List> items = new ArrayList<>(); + for (String part : parts) { + if (this.conventionService.isNullOrEmpty(part)) continue; + part = part.trim().replace(",", "").trim(); + if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue; + Map ref = new HashMap<>(); + ref.put("name", part); + ref.put("source", source); + items.add(ref); } - if (parts != null) { - List> items = new ArrayList<>(); - for (String part : parts) { - if (this.conventionService.isNullOrEmpty(part)) continue; - part = part.trim().replace(",", "").trim(); - if (this.conventionService.isNullOrEmpty(part) || part.equals("null")) continue; - Map ref = new HashMap<>(); - ref.put("name", part); - ref.put("source", source); - items.add(ref); - } - if (items.isEmpty()) break; - references = items.toArray(new Map[items.size()]); - } - } + if (items.isEmpty()) break; + references = items.toArray(new Map[items.size()]); + } } } - if (references == null && !ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())){ + if (references == null){ if (textValue.equalsIgnoreCase("[\"dublin core\",\"{\\\"id\\\":\\\"metadataschema:rd-alliance/dublin-core\\\",\\\"label\\\":\\\"Dublin Core\\\",\\\"source\\\":\\\"\\\",\\\"uri\\\":\\\"http://dublincore.org\\\"}\"]")){ Map ref = new HashMap<>(); @@ -733,8 +740,7 @@ public class DatasetMigrationService { if (this.conventionService.isNullOrEmpty(data.getLabel())) data.setLabel(data.getReference()); if (this.conventionService.isNullOrEmpty(data.getSource()) && - referenceTypeDefinition.getSources().size() == 1 && - !ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) { + referenceTypeDefinition.getSources().size() == 1) { data.setSource(referenceTypeDefinition.getSources().getFirst().getKey()); } if (this.conventionService.isNullOrEmpty(data.getLabel()) || data.getLabel().equals("null")){ @@ -742,7 +748,7 @@ public class DatasetMigrationService { return null; } - if (this.conventionService.isNullOrEmpty(data.getSource())) throw new MyApplicationException("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " Source not found source for reference : " + referenceTypeDataEntity.getReferenceTypeId() + " " + this.jsonHandlingService.toJsonSafe(item)); //TODO + if (this.conventionService.isNullOrEmpty(data.getSource())) throw new MyApplicationException("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " Source not found source for reference : " + referenceTypeDataEntity.getReferenceTypeId() + " " + this.jsonHandlingService.toJsonSafe(item)); //TODO if (this.conventionService.isNullOrEmpty(data.getReference())) { data.setReference(this.queryFactory.query(ReferenceQuery.class).sources(data.getSource()).like(data.getLabel()).typeIds(data.getTypeId()).sourceTypes(ReferenceSourceType.External).collectAs(new BaseFieldSet().ensure(Reference._reference, Reference._label)) diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java index 77bd9823a..380e460ee 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java @@ -14,6 +14,7 @@ import eu.eudat.data.ReferenceTypeEntity; import eu.eudat.data.converters.enums.DatabaseEnum; import eu.eudat.model.DescriptionTemplate; import eu.eudat.query.DescriptionTemplateQuery; +import eu.eudat.query.ReferenceTypeQuery; import eu.old.eudat.logic.services.helpers.HelpersService; import eu.old.eudat.logic.utilities.builders.XmlBuilder; import eu.old.eudat.models.data.components.commons.Multiplicity; @@ -28,6 +29,7 @@ import gr.cite.tools.logging.LoggerService; import jakarta.persistence.EntityManager; import jakarta.ws.rs.NotSupportedException; import jakarta.xml.bind.JAXBException; +import org.elasticsearch.common.inject.Guice; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; @@ -75,6 +77,7 @@ public class DescriptionTemplateXmlMigrationService { int page = 0; Map referenceTypeEntityMap = new HashMap<>(); + Map knownReferenceEntities = this.queryFactory.query(ReferenceTypeQuery.class).ids(ReferenceTypeIds.KnownReferenceTypeIds).collect().stream().collect(Collectors.toMap(ReferenceTypeEntity::getId, x-> x)); List items; do { DescriptionTemplateQuery query = this.queryFactory.query(DescriptionTemplateQuery.class); @@ -99,7 +102,7 @@ public class DescriptionTemplateXmlMigrationService { logger.error("Migrate DescriptionTemplate " + item.getId() + " failed read xml"); throw new MyApplicationException("Migrate DescriptionTemplate " + item.getId() + " failed read xml"); } - item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(viewStyleModel, referenceTypeEntityMap))); + item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(viewStyleModel, referenceTypeEntityMap, knownReferenceEntities))); this.entityManager.merge(item); } @@ -115,7 +118,7 @@ public class DescriptionTemplateXmlMigrationService { this.entityManager.flush(); } - private @NotNull DefinitionEntity buildDefinitionEntity(ViewStyleModel persist, Map referenceTypeEntityMap) throws URISyntaxException { + private @NotNull DefinitionEntity buildDefinitionEntity(ViewStyleModel persist, Map referenceTypeEntityMap, Map knownReferenceEntities) throws URISyntaxException { DefinitionEntity data = new DefinitionEntity(); if (persist == null) return data; @@ -123,14 +126,14 @@ public class DescriptionTemplateXmlMigrationService { Map> sectionPerPage = mapSectionsToPage(persist); data.setPages(new ArrayList<>()); for (Page pagePersist : persist.getPages()) { - data.getPages().add(this.buildPageEntity(pagePersist, sectionPerPage.get(pagePersist.getId()), referenceTypeEntityMap)); + data.getPages().add(this.buildPageEntity(pagePersist, sectionPerPage.get(pagePersist.getId()), referenceTypeEntityMap, knownReferenceEntities)); } } return data; } - private @NotNull SectionEntity buildSectionEntity(Section persist, Map referenceTypeEntityMap) throws URISyntaxException { + private @NotNull SectionEntity buildSectionEntity(Section persist, Map referenceTypeEntityMap, Map knownReferenceEntities) throws URISyntaxException { SectionEntity data = new SectionEntity(); if (persist == null) return data; @@ -146,21 +149,21 @@ public class DescriptionTemplateXmlMigrationService { if (!this.conventionService.isListNullOrEmpty(persist.getSections())) { data.setSections(new ArrayList<>()); for (Section sectionPersist : persist.getSections()) { - data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap)); + data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap, knownReferenceEntities)); } } if (!this.conventionService.isListNullOrEmpty(persist.getFieldSets())) { data.setFieldSets(new ArrayList<>()); for (FieldSet fieldSetPersist : persist.getFieldSets()) { - data.getFieldSets().add(this.buildFieldSetEntity(fieldSetPersist, referenceTypeEntityMap)); + data.getFieldSets().add(this.buildFieldSetEntity(fieldSetPersist, referenceTypeEntityMap, knownReferenceEntities)); } } return data; } - private @NotNull FieldSetEntity buildFieldSetEntity(FieldSet persist, Map referenceTypeEntityMap) throws URISyntaxException { + private @NotNull FieldSetEntity buildFieldSetEntity(FieldSet persist, Map referenceTypeEntityMap, Map knownReferenceEntities) throws URISyntaxException { FieldSetEntity data = new FieldSetEntity(); if (persist == null) return data; @@ -180,13 +183,13 @@ public class DescriptionTemplateXmlMigrationService { if (!this.conventionService.isListNullOrEmpty(persist.getFields())) { data.setFields(new ArrayList<>()); for (Field fieldPersist : persist.getFields()) { - data.getFields().add(this.buildFieldEntity(fieldPersist, referenceTypeEntityMap)); + data.getFields().add(this.buildFieldEntity(fieldPersist, referenceTypeEntityMap, knownReferenceEntities)); } } return data; } - private @NotNull FieldEntity buildFieldEntity(Field persist, Map referenceTypeEntityMap) throws URISyntaxException { + private @NotNull FieldEntity buildFieldEntity(Field persist, Map referenceTypeEntityMap, Map knownReferenceEntities) throws URISyntaxException { FieldEntity data = new FieldEntity(); if (persist == null) return data; @@ -213,7 +216,7 @@ public class DescriptionTemplateXmlMigrationService { }).toList()); data.setIncludeInExport(persist.getExport()); if (persist.getData() != null) { - data.setData(this.buildFieldDataEntity(persist.getData(), persist.getViewStyle().getRenderStyle(), referenceTypeEntityMap)); + data.setData(this.buildFieldDataEntity(persist.getData(), persist.getViewStyle().getRenderStyle(), referenceTypeEntityMap, knownReferenceEntities)); if (persist.getViewStyle().getRenderStyle().equals("externalDatasets")){ FieldDataExternalDatasetType type = null; try { @@ -239,7 +242,7 @@ public class DescriptionTemplateXmlMigrationService { return data; } - private BaseFieldDataEntity buildFieldDataEntity(FieldData persist, String renderStyle, Map referenceTypeEntityMap) throws URISyntaxException { + private BaseFieldDataEntity buildFieldDataEntity(FieldData persist, String renderStyle, Map referenceTypeEntityMap, Map knownReferenceEntities) throws URISyntaxException { if (renderStyle.equals("checkBox")){ return this.buildLabelDataEntity(persist.getLabel(), FieldType.CHECK_BOX); } @@ -369,7 +372,7 @@ public class DescriptionTemplateXmlMigrationService { return data; } case "autocomplete":{ - ReferenceTypeEntity referenceType = this.resolveReferenceTypeEntityAutoCompleteData(((AutoCompleteData)persist), referenceTypeEntityMap); + ReferenceTypeEntity referenceType = this.resolveReferenceTypeEntityAutoCompleteData(((AutoCompleteData)persist), referenceTypeEntityMap, knownReferenceEntities); return this.buildReferenceTypeDataEntity(persist.getLabel(), ((AutoCompleteData)persist).getMultiAutoComplete(), referenceType.getId()); } } @@ -461,7 +464,7 @@ public class DescriptionTemplateXmlMigrationService { - private @NotNull PageEntity buildPageEntity(Page persist, List
sections, Map referenceTypeEntityMap) throws URISyntaxException { + private @NotNull PageEntity buildPageEntity(Page persist, List
sections, Map referenceTypeEntityMap, Map knownReferenceEntities) throws URISyntaxException { PageEntity data = new PageEntity(); if (persist == null) return data; @@ -473,7 +476,7 @@ public class DescriptionTemplateXmlMigrationService { if (!this.conventionService.isListNullOrEmpty(sections)) { data.setSections(new ArrayList<>()); for (Section sectionPersist : sections) { - data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap)); + data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap, knownReferenceEntities)); } } return data; @@ -511,10 +514,13 @@ public class DescriptionTemplateXmlMigrationService { } } - public ReferenceTypeEntity resolveReferenceTypeEntityAutoCompleteData(AutoCompleteData persist, Map referenceTypeEntityMap) throws URISyntaxException { + public ReferenceTypeEntity resolveReferenceTypeEntityAutoCompleteData(AutoCompleteData persist, Map referenceTypeEntityMap, Map knownReferenceEntities) throws URISyntaxException { ReferenceTypeDefinitionEntity definitionEntity = this.buildReferenceTypeDefinitionEntity(persist); String hash = this.referenceTypeCalculatedHash(definitionEntity); - ReferenceTypeEntity data = referenceTypeEntityMap.getOrDefault(hash, null); + ReferenceTypeEntity data = this.getReferenceTypeEntityByUrl(definitionEntity, knownReferenceEntities); + if (data == null){ + data = referenceTypeEntityMap.getOrDefault(hash, null); + } if (data == null) { data = new ReferenceTypeEntity(); data.setId(UUID.randomUUID()); @@ -528,6 +534,26 @@ public class DescriptionTemplateXmlMigrationService { } return data; } + + private ReferenceTypeEntity getReferenceTypeEntityByUrl(ReferenceTypeDefinitionEntity definitionEntity, Map knownReferenceEntities){ + if (definitionEntity == null || this.conventionService.isListNullOrEmpty(definitionEntity.getSources()) || definitionEntity.getSources().size() != 1) return null; + ExternalFetcherApiSourceConfigurationEntity externalFetcherApiSourceConfigurationEntity = (ExternalFetcherApiSourceConfigurationEntity)definitionEntity.getSources().getFirst(); + if (externalFetcherApiSourceConfigurationEntity == null || this.conventionService.isNullOrEmpty(externalFetcherApiSourceConfigurationEntity.getUrl())) return null; + + if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/license/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.License); + else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://zenodo.org/api/licenses".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.License); + else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://zenodo.org/api/licenses?q={like}".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.License); + else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("http://api.openaire.eu/search/publications".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Publication); + else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("http://api.openaire.eu/search/publications?format=json&title={like}".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Publication); + else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://services.openaire.eu/openaire/ds/searchdetails/{page}/{pageSize}?requestSortBy=id&order=ASCENDING".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Publication); + else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/datarepo/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.DataRepositories); + else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://services.openaire.eu/search/v2/api/datasources?q={like}&fq=datasourcetypeuiname".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.DataRepositories); + else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://services.openaire.eu/search/v2/api/resources?query=oaftype exact datasource and {like} and ( datasourcetypeuiid = \"pubsrepository::journal\" or datasourcetypeuiid = \"aggregator::pubsrepository::journals\")&page=0&size=60&format=json".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Journal); + else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/metadataschema/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Registries); + else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/taxonomy/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Taxonomy); + + return null; + } public String referenceTypeCalculatedHash(ReferenceTypeDefinitionEntity definitionEntity) { List sourceKeys = new ArrayList<>(); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java index 0fcc3b7b0..f10e82c0b 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java @@ -90,29 +90,29 @@ public class MigrationController { @GetMapping("all") @Transactional public boolean migrateAll() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException, NoSuchFieldException, InvalidApplicationException, TransformerException, URISyntaxException { - //Reference - this.dataRepositoryMigrationService.migrate(); - this.externalDatasetMigrationService.migrate(); - this.funderMigrationService.migrate(); - this.grantMigrationService.migrate(); - this.organizationMigrationService.migrate(); - this.projectMigrationService.migrate(); - this.registryMigrationService.migrate(); - this.researcherMigrationService.migrate(); - this.serviceMigrationService.migrate(); +// //Reference +// this.dataRepositoryMigrationService.migrate(); +// this.externalDatasetMigrationService.migrate(); +// this.funderMigrationService.migrate(); +// this.grantMigrationService.migrate(); +// this.organizationMigrationService.migrate(); +// this.projectMigrationService.migrate(); +// this.registryMigrationService.migrate(); +// this.researcherMigrationService.migrate(); +// this.serviceMigrationService.migrate(); +// +// //User +// this.userContactInfoMigrationService.migrate(); +// this.userMigrationService.migrate(); +// +// //XML recreate +// this.dmpBlueprintXmlMigrationService.migrate(); +// this.descriptionTemplateXmlMigrationService.migrate(); - //User - this.userContactInfoMigrationService.migrate(); - this.userMigrationService.migrate(); - - //XML recreate - this.dmpBlueprintXmlMigrationService.migrate(); - this.descriptionTemplateXmlMigrationService.migrate(); - - //Dmp - this.dmpMigrationService.migrate(); - this.dmpDatasetProfileMigrationService.migrate(); - this.dmpUserMigrationService.migrate(); +// //Dmp +// this.dmpMigrationService.migrate(); +// this.dmpDatasetProfileMigrationService.migrate(); +// this.dmpUserMigrationService.migrate(); //Description this.datasetMigrationService.migrate();