MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

migration changes

This commit is contained in:
Efstratios Giannopoulos 2024-04-10 12:10:35 +03:00
parent e3479eb313
commit 53a04ba970
4 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dmp-db-scema/updates/00.01.060_Insert_values.sql
View File

@ -30,7 +30,7 @@ BEGIN
INSERT INTO public."ReferenceType" VALUES ('8ec7556b-749d-4c4a-a4b9-43d064693795', 'journals', 'journals', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><definition><sources><api><key>openaire</key><label>OpenAIRE Journals</label><ordinal>1</ordinal><referenceTypeDependencyIds/><type>0</type><authentication><authUrlMethod>0</authUrlMethod><authUrlBody></authUrlBody><authTokenPath></authTokenPath><authUrl></authUrl><enabled>false</enabled><authType></authType></authentication><contentType>application/json</contentType><filterType>remote</filterType><firstPage>0</firstPage><requestHttpMethod>1</requestHttpMethod><paginationPath>$[''meta''][''pagination''][''page'',''pages'',''count'']</paginationPath><queries><query><cases><case><value>{like}</value></case></cases><name>like</name></query></queries><requestBody>{"officialname": "{like}", "typology":"journal"}</requestBody><results><fieldsMapping><field><code>reference_id</code><responsePath>id</responsePath></field><field><code>label</code><responsePath>officialname</responsePath></field><field><code>description</code><responsePath>officialname</responsePath></field></fieldsMapping><resultsArrayPath>$[''datasourceInfo''][*]</resultsArrayPath></results><url>https://services.openaire.eu/openaire/ds/searchdetails/{page}/{pageSize}?requestSortBy=id&amp;order=ASCENDING</url></api></sources></definition>', 1, '2023-11-16 17:40:12.811667', '2024-02-16 09:09:22.816978', NULL);
INSERT INTO public."ReferenceType" VALUES ('9ec2000d-95c7-452e-b356-755fc8e2574c', 'services', 'services', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><definition><fields><field code="uri" label="Service Uri" description="Service Uri" dataType="Text"/></fields><sources><api><key>eosc</key><label>EOSC Providers</label><ordinal>1</ordinal><referenceTypeDependencyIds/><type>0</type><authentication><authUrlMethod>0</authUrlMethod><authUrlBody></authUrlBody><authTokenPath></authTokenPath><authUrl></authUrl><enabled>false</enabled><authType></authType></authentication><contentType>application/json; charset=utf-8</contentType><filterType>remote</filterType><firstPage>0</firstPage><requestHttpMethod>0</requestHttpMethod><paginationPath>$[''from'',''to'',''total'']</paginationPath><queries><query><cases><case><value>{like}</value></case></cases><name>like</name></query></queries><requestBody></requestBody><results><fieldsMapping><field><code>reference_id</code><responsePath>id</responsePath></field><field><code>label</code><responsePath>name</responsePath></field><field><code>description</code><responsePath>description</responsePath></field><field><code>uri</code><responsePath>webpage</responsePath></field></fieldsMapping><resultsArrayPath>$[''results''][*]</resultsArrayPath></results><url>https://providers.eosc-portal.eu/api/service/all?query={like}&amp;from={page}&amp;quantity={pageSize}</url></api></sources></definition>', 1, '2023-11-16 17:57:22.081053', '2024-02-16 09:07:13.944104', NULL);
INSERT INTO public."ReferenceType" VALUES ('a3ce0fb2-d72c-48bb-b322-7401940cb802', 'datasets', 'datasets', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><definition><fields><field code="pid" label="Pid Datasets" description="Pid Datasets" dataType="Text"/><field code="pidTypeField" label="PidTypeField Datasets" description="PidTypeField Datasets" dataType="Text"/></fields><sources><api><key>openaire</key><label>OpenAIRE</label><ordinal>1</ordinal><referenceTypeDependencyIds/><type>0</type><authentication><authUrlMethod>0</authUrlMethod><authUrlBody></authUrlBody><authTokenPath></authTokenPath><authUrl></authUrl><enabled>false</enabled><authType></authType></authentication><contentType>application/json; charset=utf-8</contentType><filterType>remote</filterType><firstPage>0</firstPage><requestHttpMethod>0</requestHttpMethod><paginationPath>$[''meta''][''pagination''][''page'',''pages'',''count'']</paginationPath><queries><query><cases><case><value>{like}</value></case></cases><name>like</name></query></queries><requestBody></requestBody><results><fieldsMapping><field><code>reference_id</code><responsePath>originalId</responsePath></field><field><code>label</code><responsePath>title</responsePath></field><field><code>description</code><responsePath>title</responsePath></field><field><code>pid</code><responsePath>[''pid''][''content'']</responsePath></field><field><code>pidTypeField</code><responsePath>[''pid''][''classid'']</responsePath></field></fieldsMapping><resultsArrayPath>$[''results''][*][''result''][''metadata''][''oaf:entity''][''oaf:result'']</resultsArrayPath></results><url>https://services.openaire.eu/search/v2/api/datasets/?q={like}&amp;page={page}&amp;size={pageSize}&amp;format=json</url></api></sources></definition>', 1, '2023-11-17 10:26:55.332111', '2024-02-16 15:33:35.025025', NULL);
INSERT INTO public."ReferenceType" VALUES ('ab7cdd93-bea2-440d-880d-3846dad80b21', 'taxonomies', 'taxonomies', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><definition><sources><static><key>taxonomy</key><label>Taxonomies</label><ordinal>1</ordinal><type>1</type><items><item><options><option><code>reference_id</code><value>cite_taxonomy</value></option><option><code>label</code><value>Cite Taxonom</value></option><option><code>description</code><value>Cite Taxonom</value></option></options></item></items></static></sources></definition', 1, '2023-11-16 17:48:09.769599', '2023-11-16 17:48:09.780609', NULL);
INSERT INTO public."ReferenceType" VALUES ('ab7cdd93-bea2-440d-880d-3846dad80b21', 'taxonomies', 'taxonomies', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><definition><sources><static><key>taxonomy</key><label>Taxonomies</label><ordinal>1</ordinal><type>1</type><items><item><options><option><code>reference_id</code><value>cite_taxonomy</value></option><option><code>label</code><value>Cite Taxonom</value></option><option><code>description</code><value>Cite Taxonom</value></option></options></item></items></static></sources></definition>', 1, '2023-11-16 17:48:09.769599', '2023-11-16 17:48:09.780609', NULL);
INSERT INTO public."ReferenceType" VALUES ('d26b64ba-fda0-4b14-b3ef-9856a492d55a', 'repositories', 'repositories', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><definition><sources><api><key>openaire</key><label>OpenAIRE</label><ordinal>1</ordinal><referenceTypeDependencyIds/><type>0</type><authentication><authUrlMethod>0</authUrlMethod><authUrlBody></authUrlBody><authTokenPath></authTokenPath><authUrl></authUrl><enabled>false</enabled><authType></authType></authentication><contentType>application/json</contentType><filterType>remote</filterType><firstPage>0</firstPage><requestHttpMethod>1</requestHttpMethod><paginationPath>$[''meta''][''pagination''][''page'',''pages'',''count'']</paginationPath><queries><query><cases><case><value>{like}</value></case></cases><name>like</name></query></queries><requestBody>{"officialname": "{like}", "typology":"data"}</requestBody><results><fieldsMapping><field><code>reference_id</code><responsePath>id</responsePath></field><field><code>label</code><responsePath>officialname</responsePath></field><field><code>description</code><responsePath>officialname</responsePath></field></fieldsMapping><resultsArrayPath>$[''datasourceInfo''][*]</resultsArrayPath></results><url>https://services.openaire.eu/openaire/ds/searchdetails/{page}/{pageSize}?requestSortBy=id&amp;order=ASCENDING</url></api></sources></definition>', 1, '2023-11-16 17:10:33.849688', '2024-02-16 12:45:18.66016', NULL);
INSERT INTO public."ReferenceType" VALUES ('1e927daa-b856-443f-96da-22f325f7322f', 'Publication Repositories', 'pubRepositories', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><definition><sources><api><key>openaire</key><label>OpenAIRE publication repositories 1</label><ordinal>1</ordinal><referenceTypeDependencyIds/><type>0</type><authentication><authUrlMethod>0</authUrlMethod><authUrlBody></authUrlBody><authTokenPath></authTokenPath><authUrl></authUrl><enabled>false</enabled><authType></authType></authentication><contentType>application/json</contentType><filterType>remote</filterType><firstPage>0</firstPage><requestHttpMethod>1</requestHttpMethod><paginationPath>$[''meta''][''pagination''][''page'',''pages'',''count'']</paginationPath><queries><query><cases><case><value>{like}</value></case></cases><name>like</name></query></queries><requestBody>{"officialname": "{like}", "typology":"pubsrepository::institutional"}</requestBody><results><fieldsMapping><field><code>reference_id</code><responsePath>id</responsePath></field><field><code>label</code><responsePath>officialname</responsePath></field><field><code>description</code><responsePath>officialname</responsePath></field></fieldsMapping><resultsArrayPath>$[''datasourceInfo''][*]</resultsArrayPath></results><url>https://services.openaire.eu/openaire/ds/searchdetails/{page}/{pageSize}?requestSortBy=id&amp;order=ASCENDING</url></api><api><key>openaire</key><label>OpenAIRE publication repositories 2</label><ordinal>2</ordinal><referenceTypeDependencyIds/><type>0</type><authentication><authUrlMethod>0</authUrlMethod><authUrlBody></authUrlBody><authTokenPath></authTokenPath><authUrl></authUrl><enabled>false</enabled><authType></authType></authentication><contentType>application/json</contentType><filterType>remote</filterType><firstPage>0</firstPage><requestHttpMethod>1</requestHttpMethod><paginationPath>$[''meta''][''pagination''][''page'',''pages'',''count'']</paginationPath><queries><query><cases><case><value>{like}</value></case></cases><name>like</name></query></queries><requestBody>{"officialname": "{like}", "typology":"pubsrepository::thematic"}</requestBody><results><fieldsMapping><field><code>reference_id</code><responsePath>id</responsePath></field><field><code>label</code><responsePath>officialname</responsePath></field><field><code>description</code><responsePath>count</responsePath></field></fieldsMapping><resultsArrayPath>$[''datasourceInfo''][*]</resultsArrayPath></results><url>https://services.openaire.eu/openaire/ds/searchdetails/{page}/{pageSize}?requestSortBy=id&amp;order=ASCENDING</url></api><api><key>openaire</key><label>OpenAIRE publication repositories 3</label><ordinal>3</ordinal><referenceTypeDependencyIds/><type>0</type><authentication><authUrlMethod>0</authUrlMethod><authUrlBody></authUrlBody><authTokenPath></authTokenPath><authUrl></authUrl><enabled>false</enabled><authType></authType></authentication><contentType>application/json</contentType><filterType>remote</filterType><firstPage>0</firstPage><requestHttpMethod>1</requestHttpMethod><paginationPath>$[''meta''][''pagination''][''page'',''pages'',''count'']</paginationPath><queries><query><cases><case><value>{like}</value></case></cases><name>like</name></query></queries><requestBody>{"officialname": "{like}", "typology":"pubsrepository::unknown"}</requestBody><results><fieldsMapping><field><code>reference_id</code><responsePath>id</responsePath></field><field><code>label</code><responsePath>officialname</responsePath></field><field><code>description</code><responsePath>officialname</responsePath></field></fieldsMapping><resultsArrayPath>$[''datasourceInfo''][*]</resultsArrayPath></results><url>https://services.openaire.eu/openaire/ds/searchdetails/{page}/{pageSize}?requestSortBy=id&amp;order=ASCENDING</url></api></sources></definition>', 1, '2023-11-16 17:17:40.882679', '2024-02-19 09:53:35.61152', NULL);

13
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -389,7 +389,7 @@ public class DatasetMigrationService {
fieldEntity.setTextListValue(valuesParsed == null ? List.of(textValue) : Arrays.stream(valuesParsed).toList());
}
}
case DATASET_IDENTIFIER, VALIDATION -> {
case DATASET_IDENTIFIER -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
ExternalIdentifierEntity externalIdentifierEntity = migrationTools.tryParseJsonAsObjectString(ExternalIdentifierEntity.class, textValue);
if (externalIdentifierEntity == null) throw new MyApplicationException("Could not parse dataset External Identifier : " + textValue);
@ -423,6 +423,7 @@ public class DatasetMigrationService {
case INTERNAL_ENTRIES_DMPS -> throw new MyApplicationException("Found INTERNAL_ENTRIES_DMPS into description");
case INTERNAL_ENTRIES_DESCRIPTIONS -> throw new MyApplicationException("Found INTERNAL_ENTRIES_DMPS into description");
case VALIDATION -> throw new MyApplicationException("Found VALIDATION into description");
case REFERENCE_TYPES -> {
if(!this.conventionService.isNullOrEmpty(textValue)) { //TODO
ReferenceTypeDataEntity referenceTypeDataEntity = (ReferenceTypeDataEntity)currentField.getData();
@ -446,8 +447,10 @@ public class DatasetMigrationService {
//TODO: {"name":"Euro","value":"EUR"} what we want to keep ?
Map<String, Object> reference = new HashMap<>();
reference.put(ReferenceEntity.KnownFields.ReferenceId, currency.getValue());
reference.put(ReferenceEntity.KnownFields.Label, currency.getName());
reference.put("pid", currency.getValue());
reference.put("name", currency.getName());
reference.put("key", "currencies");
references = List.of(reference).toArray(Map[]::new);
}
@ -976,7 +979,7 @@ public class DatasetMigrationService {
ReferenceEntity existingEntity = referenceMap.getOrDefault(new ReferenceKey(data), null);
if(existingEntity == null){
if (data.getCreatedById() != null) referenceMap.put(new ReferenceKey(data), data);
if (data.getCreatedById() == null) referenceMap.put(new ReferenceKey(data), data);
this.entityManager.persist(data);
return data;
} else {
@ -1074,7 +1077,7 @@ public class DatasetMigrationService {
this.type = reference.getTypeId();
this.source = reference.getSource().trim();
this.reference = reference.getReference().trim();
hashCode = Objects.hash(this.type, this.source, this.reference);
hashCode = Objects.hash(this.type, this.source.hashCode(), this.reference);
}
public UUID getType() {

1
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java
View File

@ -435,7 +435,6 @@ public class DescriptionTemplateXmlMigrationService {
}
}
case UPLOAD -> throw new NotSupportedException("Upload validator not supported");
case CURRENCY -> throw new NotSupportedException("Currency validator not supported");
case TAGS -> throw new NotSupportedException("Tags validator not supported");
case INTERNAL_ENTRIES_DMPS -> throw new NotSupportedException("INTERNAL_ENTRIES_DMPS validator not supported");

8
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpBlueprintXmlMigrationService.java
View File

@ -14,6 +14,7 @@ import eu.eudat.query.DmpBlueprintQuery;
import eu.old.eudat.logic.utilities.builders.XmlBuilder;
import eu.old.eudat.models.data.entities.xmlmodels.dmpprofiledefinition.*;
import eu.old.eudat.models.data.entities.xmlmodels.dmpprofiledefinition.types.FieldCategory;
import eu.old.eudat.models.data.entities.xmlmodels.dmpprofiledefinition.types.SystemFieldType;
import gr.cite.tools.data.query.Ordering;
import gr.cite.tools.data.query.Paging;
import gr.cite.tools.data.query.QueryFactory;
@ -190,7 +191,12 @@ public class DmpBlueprintXmlMigrationService {
case RESEARCHERS -> referenceTypeFieldEntity.setReferenceTypeId(ReferenceTypeIds.Researcher);
default -> throw new MyApplicationException("Type not found " + systemField.getType());
}
dataTyped.setRequired(systemField.isRequired());
if (systemField.getType().equals(SystemFieldType.TEXT) || systemField.getType().equals(SystemFieldType.HTML_TEXT) ||
systemField.getType().equals(SystemFieldType.LANGUAGE) || systemField.getType().equals(SystemFieldType.ACCESS_RIGHTS)){
dataTyped.setRequired(true);
} else {
dataTyped.setRequired(systemField.isRequired());
}
referenceTypeFieldEntity.setRequired(systemField.isRequired());
if (referenceTypeFieldEntity.getReferenceTypeId() == null){