migration fixes
This commit is contained in:
parent
c27e98acd8
commit
33d9bab4bf
|
@ -489,6 +489,16 @@ public class DatasetMigrationService {
|
|||
ref.put("source", "opendefinition");
|
||||
} else if (ReferenceTypeIds.PubRepositories.equals(referenceTypeDataEntity.getReferenceTypeId())) {
|
||||
ref.put("source", "opendefinition");
|
||||
} else if (ReferenceTypeIds.Grants.equals(referenceTypeDataEntity.getReferenceTypeId())) {
|
||||
ref.put("source", "openaire");
|
||||
} else if (ReferenceTypeIds.Project.equals(referenceTypeDataEntity.getReferenceTypeId())) {
|
||||
ref.put("source", "project");
|
||||
} else if (ReferenceTypeIds.Taxonomy.equals(referenceTypeDataEntity.getReferenceTypeId())) {
|
||||
ref.put("source", "taxonomy");
|
||||
} else if (ReferenceTypeIds.Publication.equals(referenceTypeDataEntity.getReferenceTypeId())) {
|
||||
ref.put("source", "openaire");
|
||||
} else if (ReferenceTypeIds.Currency.equals(referenceTypeDataEntity.getReferenceTypeId())) {
|
||||
ref.put("source", "currencies");
|
||||
} else if (ReferenceTypeIds.Researcher.equals(referenceTypeDataEntity.getReferenceTypeId())) {
|
||||
Matcher pattern = Pattern.compile("\\((.*?)\\)").matcher(textValue);
|
||||
if (pattern.find()) {
|
||||
|
@ -504,13 +514,11 @@ public class DatasetMigrationService {
|
|||
}
|
||||
if (ref.containsKey("source")) references = List.of(ref).toArray(new Map[1]);
|
||||
} else {
|
||||
String source = null;
|
||||
if (!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) {
|
||||
ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null);
|
||||
if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
|
||||
if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
|
||||
//if (referenceTypeDefinition.getSources().size() > 1) throw new MyApplicationException("Multiple reference source for reference : " + this.jsonHandlingService.toJsonSafe(textValue));
|
||||
source = referenceTypeDefinition.getSources().getFirst().getKey();
|
||||
String source = referenceTypeDefinition.getSources().getFirst().getKey();
|
||||
String[] parts = null;
|
||||
if (textValue.startsWith("[\"")){
|
||||
parts = migrationTools.tryParseJsonAsObjectString(String[].class, textValue);
|
||||
|
@ -535,9 +543,8 @@ public class DatasetMigrationService {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (references == null && !ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())){
|
||||
if (references == null){
|
||||
|
||||
if (textValue.equalsIgnoreCase("[\"dublin core\",\"{\\\"id\\\":\\\"metadataschema:rd-alliance/dublin-core\\\",\\\"label\\\":\\\"Dublin Core\\\",\\\"source\\\":\\\"\\\",\\\"uri\\\":\\\"http://dublincore.org\\\"}\"]")){
|
||||
Map<String, Object> ref = new HashMap<>();
|
||||
|
@ -733,8 +740,7 @@ public class DatasetMigrationService {
|
|||
|
||||
if (this.conventionService.isNullOrEmpty(data.getLabel())) data.setLabel(data.getReference());
|
||||
if (this.conventionService.isNullOrEmpty(data.getSource()) &&
|
||||
referenceTypeDefinition.getSources().size() == 1 &&
|
||||
!ReferenceTypeIds.KnownReferenceTypeIds.contains(referenceTypeDataEntity.getReferenceTypeId())) {
|
||||
referenceTypeDefinition.getSources().size() == 1) {
|
||||
data.setSource(referenceTypeDefinition.getSources().getFirst().getKey());
|
||||
}
|
||||
if (this.conventionService.isNullOrEmpty(data.getLabel()) || data.getLabel().equals("null")){
|
||||
|
|
|
@ -14,6 +14,7 @@ import eu.eudat.data.ReferenceTypeEntity;
|
|||
import eu.eudat.data.converters.enums.DatabaseEnum;
|
||||
import eu.eudat.model.DescriptionTemplate;
|
||||
import eu.eudat.query.DescriptionTemplateQuery;
|
||||
import eu.eudat.query.ReferenceTypeQuery;
|
||||
import eu.old.eudat.logic.services.helpers.HelpersService;
|
||||
import eu.old.eudat.logic.utilities.builders.XmlBuilder;
|
||||
import eu.old.eudat.models.data.components.commons.Multiplicity;
|
||||
|
@ -28,6 +29,7 @@ import gr.cite.tools.logging.LoggerService;
|
|||
import jakarta.persistence.EntityManager;
|
||||
import jakarta.ws.rs.NotSupportedException;
|
||||
import jakarta.xml.bind.JAXBException;
|
||||
import org.elasticsearch.common.inject.Guice;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.http.MediaType;
|
||||
|
@ -75,6 +77,7 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
int page = 0;
|
||||
|
||||
Map<String, ReferenceTypeEntity> referenceTypeEntityMap = new HashMap<>();
|
||||
Map<UUID, ReferenceTypeEntity> knownReferenceEntities = this.queryFactory.query(ReferenceTypeQuery.class).ids(ReferenceTypeIds.KnownReferenceTypeIds).collect().stream().collect(Collectors.toMap(ReferenceTypeEntity::getId, x-> x));
|
||||
List<DescriptionTemplateEntity> items;
|
||||
do {
|
||||
DescriptionTemplateQuery query = this.queryFactory.query(DescriptionTemplateQuery.class);
|
||||
|
@ -99,7 +102,7 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
logger.error("Migrate DescriptionTemplate " + item.getId() + " failed read xml");
|
||||
throw new MyApplicationException("Migrate DescriptionTemplate " + item.getId() + " failed read xml");
|
||||
}
|
||||
item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(viewStyleModel, referenceTypeEntityMap)));
|
||||
item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(viewStyleModel, referenceTypeEntityMap, knownReferenceEntities)));
|
||||
this.entityManager.merge(item);
|
||||
}
|
||||
|
||||
|
@ -115,7 +118,7 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
this.entityManager.flush();
|
||||
}
|
||||
|
||||
private @NotNull DefinitionEntity buildDefinitionEntity(ViewStyleModel persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
|
||||
private @NotNull DefinitionEntity buildDefinitionEntity(ViewStyleModel persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
|
||||
DefinitionEntity data = new DefinitionEntity();
|
||||
if (persist == null)
|
||||
return data;
|
||||
|
@ -123,14 +126,14 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
Map<String, List<Section>> sectionPerPage = mapSectionsToPage(persist);
|
||||
data.setPages(new ArrayList<>());
|
||||
for (Page pagePersist : persist.getPages()) {
|
||||
data.getPages().add(this.buildPageEntity(pagePersist, sectionPerPage.get(pagePersist.getId()), referenceTypeEntityMap));
|
||||
data.getPages().add(this.buildPageEntity(pagePersist, sectionPerPage.get(pagePersist.getId()), referenceTypeEntityMap, knownReferenceEntities));
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private @NotNull SectionEntity buildSectionEntity(Section persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
|
||||
private @NotNull SectionEntity buildSectionEntity(Section persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
|
||||
SectionEntity data = new SectionEntity();
|
||||
if (persist == null)
|
||||
return data;
|
||||
|
@ -146,21 +149,21 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
if (!this.conventionService.isListNullOrEmpty(persist.getSections())) {
|
||||
data.setSections(new ArrayList<>());
|
||||
for (Section sectionPersist : persist.getSections()) {
|
||||
data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap));
|
||||
data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap, knownReferenceEntities));
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.conventionService.isListNullOrEmpty(persist.getFieldSets())) {
|
||||
data.setFieldSets(new ArrayList<>());
|
||||
for (FieldSet fieldSetPersist : persist.getFieldSets()) {
|
||||
data.getFieldSets().add(this.buildFieldSetEntity(fieldSetPersist, referenceTypeEntityMap));
|
||||
data.getFieldSets().add(this.buildFieldSetEntity(fieldSetPersist, referenceTypeEntityMap, knownReferenceEntities));
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private @NotNull FieldSetEntity buildFieldSetEntity(FieldSet persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
|
||||
private @NotNull FieldSetEntity buildFieldSetEntity(FieldSet persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
|
||||
FieldSetEntity data = new FieldSetEntity();
|
||||
if (persist == null)
|
||||
return data;
|
||||
|
@ -180,13 +183,13 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
if (!this.conventionService.isListNullOrEmpty(persist.getFields())) {
|
||||
data.setFields(new ArrayList<>());
|
||||
for (Field fieldPersist : persist.getFields()) {
|
||||
data.getFields().add(this.buildFieldEntity(fieldPersist, referenceTypeEntityMap));
|
||||
data.getFields().add(this.buildFieldEntity(fieldPersist, referenceTypeEntityMap, knownReferenceEntities));
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
private @NotNull FieldEntity buildFieldEntity(Field persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
|
||||
private @NotNull FieldEntity buildFieldEntity(Field persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
|
||||
FieldEntity data = new FieldEntity();
|
||||
if (persist == null)
|
||||
return data;
|
||||
|
@ -213,7 +216,7 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
}).toList());
|
||||
data.setIncludeInExport(persist.getExport());
|
||||
if (persist.getData() != null) {
|
||||
data.setData(this.buildFieldDataEntity(persist.getData(), persist.getViewStyle().getRenderStyle(), referenceTypeEntityMap));
|
||||
data.setData(this.buildFieldDataEntity(persist.getData(), persist.getViewStyle().getRenderStyle(), referenceTypeEntityMap, knownReferenceEntities));
|
||||
if (persist.getViewStyle().getRenderStyle().equals("externalDatasets")){
|
||||
FieldDataExternalDatasetType type = null;
|
||||
try {
|
||||
|
@ -239,7 +242,7 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
return data;
|
||||
}
|
||||
|
||||
private BaseFieldDataEntity buildFieldDataEntity(FieldData<?> persist, String renderStyle, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
|
||||
private BaseFieldDataEntity buildFieldDataEntity(FieldData<?> persist, String renderStyle, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
|
||||
if (renderStyle.equals("checkBox")){
|
||||
return this.buildLabelDataEntity(persist.getLabel(), FieldType.CHECK_BOX);
|
||||
}
|
||||
|
@ -369,7 +372,7 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
return data;
|
||||
}
|
||||
case "autocomplete":{
|
||||
ReferenceTypeEntity referenceType = this.resolveReferenceTypeEntityAutoCompleteData(((AutoCompleteData)persist), referenceTypeEntityMap);
|
||||
ReferenceTypeEntity referenceType = this.resolveReferenceTypeEntityAutoCompleteData(((AutoCompleteData)persist), referenceTypeEntityMap, knownReferenceEntities);
|
||||
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((AutoCompleteData)persist).getMultiAutoComplete(), referenceType.getId());
|
||||
}
|
||||
}
|
||||
|
@ -461,7 +464,7 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
|
||||
|
||||
|
||||
private @NotNull PageEntity buildPageEntity(Page persist, List<Section> sections, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
|
||||
private @NotNull PageEntity buildPageEntity(Page persist, List<Section> sections, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
|
||||
PageEntity data = new PageEntity();
|
||||
if (persist == null)
|
||||
return data;
|
||||
|
@ -473,7 +476,7 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
if (!this.conventionService.isListNullOrEmpty(sections)) {
|
||||
data.setSections(new ArrayList<>());
|
||||
for (Section sectionPersist : sections) {
|
||||
data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap));
|
||||
data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap, knownReferenceEntities));
|
||||
}
|
||||
}
|
||||
return data;
|
||||
|
@ -511,10 +514,13 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
}
|
||||
}
|
||||
|
||||
public ReferenceTypeEntity resolveReferenceTypeEntityAutoCompleteData(AutoCompleteData persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
|
||||
public ReferenceTypeEntity resolveReferenceTypeEntityAutoCompleteData(AutoCompleteData persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap, Map<UUID, ReferenceTypeEntity> knownReferenceEntities) throws URISyntaxException {
|
||||
ReferenceTypeDefinitionEntity definitionEntity = this.buildReferenceTypeDefinitionEntity(persist);
|
||||
String hash = this.referenceTypeCalculatedHash(definitionEntity);
|
||||
ReferenceTypeEntity data = referenceTypeEntityMap.getOrDefault(hash, null);
|
||||
ReferenceTypeEntity data = this.getReferenceTypeEntityByUrl(definitionEntity, knownReferenceEntities);
|
||||
if (data == null){
|
||||
data = referenceTypeEntityMap.getOrDefault(hash, null);
|
||||
}
|
||||
if (data == null) {
|
||||
data = new ReferenceTypeEntity();
|
||||
data.setId(UUID.randomUUID());
|
||||
|
@ -529,6 +535,26 @@ public class DescriptionTemplateXmlMigrationService {
|
|||
return data;
|
||||
}
|
||||
|
||||
private ReferenceTypeEntity getReferenceTypeEntityByUrl(ReferenceTypeDefinitionEntity definitionEntity, Map<UUID, ReferenceTypeEntity> knownReferenceEntities){
|
||||
if (definitionEntity == null || this.conventionService.isListNullOrEmpty(definitionEntity.getSources()) || definitionEntity.getSources().size() != 1) return null;
|
||||
ExternalFetcherApiSourceConfigurationEntity externalFetcherApiSourceConfigurationEntity = (ExternalFetcherApiSourceConfigurationEntity)definitionEntity.getSources().getFirst();
|
||||
if (externalFetcherApiSourceConfigurationEntity == null || this.conventionService.isNullOrEmpty(externalFetcherApiSourceConfigurationEntity.getUrl())) return null;
|
||||
|
||||
if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/license/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.License);
|
||||
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://zenodo.org/api/licenses".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.License);
|
||||
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://zenodo.org/api/licenses?q={like}".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.License);
|
||||
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("http://api.openaire.eu/search/publications".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Publication);
|
||||
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("http://api.openaire.eu/search/publications?format=json&title={like}".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Publication);
|
||||
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://services.openaire.eu/openaire/ds/searchdetails/{page}/{pageSize}?requestSortBy=id&order=ASCENDING".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Publication);
|
||||
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/datarepo/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.DataRepositories);
|
||||
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://services.openaire.eu/search/v2/api/datasources?q={like}&fq=datasourcetypeuiname".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.DataRepositories);
|
||||
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://services.openaire.eu/search/v2/api/resources?query=oaftype exact datasource and {like} and ( datasourcetypeuiid = \"pubsrepository::journal\" or datasourcetypeuiid = \"aggregator::pubsrepository::journals\")&page=0&size=60&format=json".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Journal);
|
||||
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/metadataschema/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Registries);
|
||||
else if (externalFetcherApiSourceConfigurationEntity.getUrl().trim().toLowerCase(Locale.ROOT).equals("https://eestore.paas2.uninett.no/api/taxonomy/".trim().toLowerCase(Locale.ROOT))) return knownReferenceEntities.get(ReferenceTypeIds.Taxonomy);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public String referenceTypeCalculatedHash(ReferenceTypeDefinitionEntity definitionEntity) {
|
||||
List<String> sourceKeys = new ArrayList<>();
|
||||
if (definitionEntity.getSources() != null){
|
||||
|
|
|
@ -90,29 +90,29 @@ public class MigrationController {
|
|||
@GetMapping("all")
|
||||
@Transactional
|
||||
public boolean migrateAll() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException, NoSuchFieldException, InvalidApplicationException, TransformerException, URISyntaxException {
|
||||
//Reference
|
||||
this.dataRepositoryMigrationService.migrate();
|
||||
this.externalDatasetMigrationService.migrate();
|
||||
this.funderMigrationService.migrate();
|
||||
this.grantMigrationService.migrate();
|
||||
this.organizationMigrationService.migrate();
|
||||
this.projectMigrationService.migrate();
|
||||
this.registryMigrationService.migrate();
|
||||
this.researcherMigrationService.migrate();
|
||||
this.serviceMigrationService.migrate();
|
||||
// //Reference
|
||||
// this.dataRepositoryMigrationService.migrate();
|
||||
// this.externalDatasetMigrationService.migrate();
|
||||
// this.funderMigrationService.migrate();
|
||||
// this.grantMigrationService.migrate();
|
||||
// this.organizationMigrationService.migrate();
|
||||
// this.projectMigrationService.migrate();
|
||||
// this.registryMigrationService.migrate();
|
||||
// this.researcherMigrationService.migrate();
|
||||
// this.serviceMigrationService.migrate();
|
||||
//
|
||||
// //User
|
||||
// this.userContactInfoMigrationService.migrate();
|
||||
// this.userMigrationService.migrate();
|
||||
//
|
||||
// //XML recreate
|
||||
// this.dmpBlueprintXmlMigrationService.migrate();
|
||||
// this.descriptionTemplateXmlMigrationService.migrate();
|
||||
|
||||
//User
|
||||
this.userContactInfoMigrationService.migrate();
|
||||
this.userMigrationService.migrate();
|
||||
|
||||
//XML recreate
|
||||
this.dmpBlueprintXmlMigrationService.migrate();
|
||||
this.descriptionTemplateXmlMigrationService.migrate();
|
||||
|
||||
//Dmp
|
||||
this.dmpMigrationService.migrate();
|
||||
this.dmpDatasetProfileMigrationService.migrate();
|
||||
this.dmpUserMigrationService.migrate();
|
||||
// //Dmp
|
||||
// this.dmpMigrationService.migrate();
|
||||
// this.dmpDatasetProfileMigrationService.migrate();
|
||||
// this.dmpUserMigrationService.migrate();
|
||||
|
||||
//Description
|
||||
this.datasetMigrationService.migrate();
|
||||
|
|
Loading…
Reference in New Issue