Merge branch 'reference-type-refactor' into dmp-refactoring

# Conflicts:
#	dmp-frontend/src/app/ui/admin/description-template/editor/components/field/description-template-editor-field.component.html
#	dmp-frontend/src/app/ui/description/editor/description-form/components/form-field/form-field.component.html
#	dmp-frontend/src/app/ui/reference/reference-field/reference-field.component.html
#	dmp-frontend/src/app/ui/reference/reference-field/reference-field.component.ts

dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java

@@ -23,6 +23,7 @@ public final class Permission {
 	public static String PublicBrowseUser = "PublicBrowseUser";
 	public static String PublicBrowseDashboardStatistics = "PublicBrowseDashboardStatistics";
 	public static String PublicSendContactSupport = "PublicSendContactSupport";
+	public static String PublicBrowseReferenceType = "PublicBrowseReferenceType";
 	//Elastic
 	public static String ManageElastic = "ManageElastic";
 

dmp-backend/core/src/main/java/eu/eudat/commons/enums/DmpBlueprintFieldCategory.java

@@ -7,11 +7,13 @@ import java.util.Map;
 
 public enum DmpBlueprintFieldCategory implements DatabaseEnum<Short> {
     System((short) 0),
-    Extra((short) 1);
+    Extra((short) 1),
+    ReferenceType((short) 1);
 
     public static class Names {
         public static final String System = "system";
         public static final String Extra = "extra";
+        public static final String ReferenceType = "referenceType";
     }
 
     private final Short value;

dmp-backend/core/src/main/java/eu/eudat/commons/enums/DmpBlueprintSystemFieldType.java

@@ -7,17 +7,11 @@ import java.util.Map;
 
 public enum DmpBlueprintSystemFieldType implements DatabaseEnum<Short> {
 
-    Text((short)0), //TODO: Rename to "Title"
+    Title((short)0),
-    HtmlText((short)1), //TODO: Rename to "Description"
+    Description((short)1),
-    Researchers((short)2), //TODO: replace by reference type
+    Language((short)2),
-    Organizations((short)3), //TODO: replace by reference type
+    Contact((short)3),
-    Language((short)4),
+    AccessRights((short)4);
-    Contact((short)5),
-    Funder((short)6), //TODO: replace by reference type
-    Grant((short)7), //TODO: replace by reference type
-    Project((short)8), //TODO: replace by reference type
-    License((short)9), //TODO: replace by reference type
-    AccessRights((short)10);
     private final Short value;
 
     DmpBlueprintSystemFieldType(Short value) {

dmp-backend/core/src/main/java/eu/eudat/commons/enums/FieldType.java

@@ -6,42 +6,29 @@ import eu.eudat.data.converters.enums.DatabaseEnum;
 import java.util.Map;
 
 public enum FieldType implements DatabaseEnum<String> {
-	EXTERNAL_SELECT(Names.ExternalSelect), //TODO: remove
 	SELECT(Names.Select),
 	BOOLEAN_DECISION(Names.BooleanDecision),
 	RADIO_BOX(Names.RadioBox),
-	INTERNAL_DMP_ENTRIES_RESEARCHERS(Names.InternalDmpResearchers), //TODO: remove
 	INTERNAL_DMP_ENTRIES_DMPS(Names.InternalDmpDmps), //TODO: rename InternalEntitiesDmps
-	INTERNAL_DMP_ENTRIES_DATASETS(Names.InternalDmpDescriptions),
+	INTERNAL_DMP_ENTRIES_DATASETS(Names.InternalDmpDescriptions),//TODO: rename InternalEntitiesDescriptions
 	CHECK_BOX(Names.CheckBox),
 	FREE_TEXT(Names.FreeText),
 	TEXT_AREA(Names.TextArea),
 	RICH_TEXT_AREA(Names.RichTextarea),
 	UPLOAD(Names.Upload),
 	DATE_PICKER(Names.DatePicker),
-	EXTERNAL_DATASETS(Names.ExternalDatasets), //TODO: replace with reference type
-	DATA_REPOSITORIES(Names.DataRepositories), //TODO: replace with reference type
-	JOURNAL_REPOSITORIES(Names.JournalRepositories), //TODO: replace with reference type
-	PUB_REPOSITORIES(Names.PubRepositories), //TODO: replace with reference type
-	LICENSES(Names.Licenses), //TODO: replace with reference type
-	TAXONOMIES(Names.Taxonomies), //TODO: replace with reference type
-	PUBLICATIONS(Names.Publications), //TODO: replace with reference type
-	REGISTRIES(Names.Registries), //TODO: replace with reference type
-	SERVICES(Names.Services), //TODO: replace with reference type
 	TAGS(Names.Tags),
-	RESEARCHERS(Names.Researchers), //TODO: replace with reference type
+	EXTERNAL_DATASETS(Names.ExternalDatasets),
-	ORGANIZATIONS(Names.Organizations), //TODO: replace with reference type
+	REFERENCE_TYPES(Names.ReferenceTypes),
 	DATASET_IDENTIFIER(Names.DatasetIdentifier),
 	CURRENCY(Names.Currency),
 	VALIDATION(Names.Validation);
 	private final String value;
 	
 	public static class Names {
-		public static final String ExternalSelect = "externalSelect";
 		public static final String Select = "select";
 		public static final String BooleanDecision = "booleanDecision";
 		public static final String RadioBox = "radiobox";
-		public static final String InternalDmpResearchers = "internalDmpResearchers";
 		public static final String InternalDmpDmps = "internalDmpDmps";
 		public static final String InternalDmpDescriptions = "internalDmpDatasets";
 		public static final String CheckBox = "checkBox";
@@ -50,21 +37,12 @@ public enum FieldType implements DatabaseEnum<String> {
 		public static final String RichTextarea = "richTextarea";
 		public static final String Upload = "upload";
 		public static final String DatePicker = "datePicker";
-		public static final String ExternalDatasets = "externalDatasets";
-		public static final String DataRepositories = "dataRepositories";
-		public static final String JournalRepositories = "journalRepositories";
-		public static final String PubRepositories = "pubRepositories";
-		public static final String Licenses = "licenses";
-		public static final String Taxonomies = "taxonomies";
-		public static final String Publications = "publications";
-		public static final String Registries = "registries";
-		public static final String Services = "services";
 		public static final String Tags = "tags";
-		public static final String Researchers = "researchers";
-		public static final String Organizations = "organizations";
 		public static final String DatasetIdentifier = "datasetIdentifier";
 		public static final String Currency = "currency";
 		public static final String Validation = "validation";
+		public static final String ReferenceTypes = "referenceTypes";
+		public static final String ExternalDatasets = "externalDatasets";
 	}
 
 	FieldType(String value) {
@@ -82,15 +60,9 @@ public enum FieldType implements DatabaseEnum<String> {
 		return map.get(i);
 	}
 
-	public static boolean isNotImplemented(FieldType fieldType){
-		return fieldType.equals(FieldType.EXTERNAL_SELECT);
-	}
 
 	public static boolean isReferenceType(FieldType fieldType){
-		return fieldType.equals(FieldType.ORGANIZATIONS) || fieldType.equals(FieldType.RESEARCHERS) || fieldType.equals(FieldType.SERVICES) ||
+		return fieldType.equals(FieldType.REFERENCE_TYPES);
-				fieldType.equals(FieldType.REGISTRIES) || fieldType.equals(FieldType.PUBLICATIONS) || fieldType.equals(FieldType.TAXONOMIES)||
-				fieldType.equals(FieldType.LICENSES) || fieldType.equals(FieldType.DATA_REPOSITORIES) || fieldType.equals(FieldType.PUB_REPOSITORIES)||
-				fieldType.equals(FieldType.JOURNAL_REPOSITORIES) || fieldType.equals(FieldType.EXTERNAL_DATASETS);
 	}
 
 	public static boolean isTextType(FieldType fieldType){
@@ -101,7 +73,7 @@ public enum FieldType implements DatabaseEnum<String> {
 	}
 
 	public static boolean isTextListType(FieldType fieldType){
-		return  fieldType.equals(FieldType.SELECT) || fieldType.equals(FieldType.TAGS) || fieldType.equals(FieldType.INTERNAL_DMP_ENTRIES_RESEARCHERS) || fieldType.equals(FieldType.INTERNAL_DMP_ENTRIES_DMPS) ||
+		return  fieldType.equals(FieldType.SELECT) || fieldType.equals(FieldType.TAGS)  || fieldType.equals(FieldType.INTERNAL_DMP_ENTRIES_DMPS) ||
 				fieldType.equals(FieldType.INTERNAL_DMP_ENTRIES_DATASETS);
 	}
 

dmp-backend/core/src/main/java/eu/eudat/commons/enums/ReferenceType.java

@@ -1,39 +0,0 @@
-package eu.eudat.commons.enums;
-
-import com.fasterxml.jackson.annotation.JsonValue;
-import eu.eudat.data.converters.enums.DatabaseEnum;
-
-import java.util.Map;
-
-public enum ReferenceType implements DatabaseEnum<Short> {
-	Taxonomies((short) 0),
-	Licenses((short) 1),
-	Publications((short) 2),
-	Journals((short) 3),
-	PubRepositories((short) 4),
-	DataRepositories((short) 5),
-	Registries((short) 6),
-	Services((short) 7),
-	Project((short) 8),
-	Funder((short) 9),
-	Datasets((short) 10),
-	Organizations((short) 11),
-	Grants((short) 12),
-	Researcher((short) 13);
-	private final Short value;
-
-	ReferenceType(Short value) {
-		this.value = value;
-	}
-
-	@JsonValue
-	public Short getValue() {
-		return value;
-	}
-
-	private static final Map<Short, ReferenceType> map = EnumUtils.getEnumValueMap(ReferenceType.class);
-
-	public static ReferenceType of(Short i) {
-		return map.get(i);
-	}
-}

dmp-backend/core/src/main/java/eu/eudat/commons/types/descriptiontemplate/FieldEntity.java

@@ -30,7 +30,7 @@ public class FieldEntity {
             @XmlElement(name = UploadDataEntity.XmlElementName, type = UploadDataEntity.class),
             @XmlElement(name = RadioBoxDataEntity.XmlElementName, type = RadioBoxDataEntity.class),
             @XmlElement(name = SelectDataEntity.XmlElementName, type = SelectDataEntity.class),
-            @XmlElement(name = ExternalSelectDataEntity.XmlElementName, type = ExternalSelectDataEntity.class),
+            @XmlElement(name = ReferenceTypeDataEntity.XmlElementName, type = ReferenceTypeDataEntity.class)
     })
     private BaseFieldDataEntity data;
     @XmlElementWrapper(name = "validations")

dmp-backend/core/src/main/java/eu/eudat/commons/types/descriptiontemplate/fielddata/ExternalSelectDataEntity.java

@@ -1,171 +0,0 @@
-package eu.eudat.commons.types.descriptiontemplate.fielddata;
-
-import jakarta.xml.bind.annotation.*;
-
-import java.util.List;
-
-
-@XmlAccessorType(XmlAccessType.FIELD)
-public class ExternalSelectDataEntity extends LabelAndMultiplicityDataEntity {
-    public static final String XmlElementName = "externalSelectData";
-
-    @XmlAccessorType(XmlAccessType.FIELD)
-    public static class ExternalSelectAuthDataEntity {
-        @XmlAttribute(name="url")
-        private String url;
-        @XmlAttribute(name="method")
-        private String method;
-        @XmlAttribute(name="body")
-        private String body;
-        @XmlAttribute(name="path")
-        private String path;
-        @XmlAttribute(name="type")
-        private String type;
-
-        public String getUrl() {
-            return url;
-        }
-
-        public void setUrl(String url) {
-            this.url = url;
-        }
-
-        public String getMethod() {
-            return method;
-        }
-
-        public void setMethod(String method) {
-            this.method = method;
-        }
-
-        public String getBody() {
-            return body;
-        }
-
-        public void setBody(String body) {
-            this.body = body;
-        }
-
-        public String getPath() {
-            return path;
-        }
-
-        public void setPath(String path) {
-            this.path = path;
-        }
-
-        public String getType() {
-            return type;
-        }
-
-        public void setType(String type) {
-            this.type = type;
-        }
-    }
-
-    @XmlAccessorType(XmlAccessType.FIELD)
-    public static class ExternalSelectSourceBindingEntity {
-        @XmlAttribute(name="label")
-        private String label;
-        @XmlAttribute(name="value")
-        private String value;
-        @XmlAttribute(name="source")
-        private String source;
-
-        public String getLabel() {
-            return label;
-        }
-        public void setLabel(String label) {
-            this.label = label;
-        }
-
-        public String getValue() {
-            return value;
-        }
-        public void setValue(String value) {
-            this.value = value;
-        }
-
-        public String getSource() {
-            return source;
-        }
-        public void setSource(String source) {
-            this.source = source;
-        }
-    }
-
-    @XmlAccessorType(XmlAccessType.FIELD)
-    public static class ExternalSelectSourceEntity {
-        @XmlAttribute(name="url")
-        private String url;
-        @XmlAttribute(name="method")
-        private String method;
-        @XmlAttribute(name="optionsRoot")
-        private String optionsRoot;
-        @XmlElement(name="sourceBinding")
-        private ExternalSelectSourceBindingEntity sourceBinding;
-        @XmlAttribute(name="hasAuth")
-        private Boolean hasAuth;
-        @XmlElement(name="auth")
-        private ExternalSelectAuthDataEntity auth;
-
-        public String getOptionsRoot() {
-            return optionsRoot;
-        }
-        public void setOptionsRoot(String optionsRoot) {
-            this.optionsRoot = optionsRoot;
-        }
-
-        public String getUrl() {
-            return url;
-        }
-        public void setUrl(String url) {
-            this.url = url;
-        }
-
-        public Boolean getHasAuth() {
-            return hasAuth;
-        }
-
-        public void setHasAuth(Boolean hasAuth) {
-            this.hasAuth = hasAuth;
-        }
-
-        public ExternalSelectAuthDataEntity getAuth() {
-            return auth;
-        }
-
-        public void setAuth(ExternalSelectAuthDataEntity auth) {
-            this.auth = auth;
-        }
-
-        public ExternalSelectSourceBindingEntity getSourceBinding() {
-            return sourceBinding;
-        }
-        public void setSourceBinding(ExternalSelectSourceBindingEntity sourceBinding) {
-            this.sourceBinding = sourceBinding;
-        }
-
-        public String getMethod() {
-            return method;
-        }
-
-        public void setMethod(String method) {
-            this.method = method;
-        }
-    }
-
-    @XmlElementWrapper(name = "sources")
-    @XmlElement(name = "source")
-    private List<ExternalSelectSourceEntity> sources;
-
-    public List<ExternalSelectSourceEntity> getSources() {
-        return sources;
-    }
-
-    public void setSources(List<ExternalSelectSourceEntity> externalSelectSourceEntityList) {
-        this.sources = externalSelectSourceEntityList;
-    }
-}
-
-

dmp-backend/core/src/main/java/eu/eudat/commons/types/descriptiontemplate/fielddata/ReferenceTypeDataEntity.java

@@ -0,0 +1,33 @@
+package eu.eudat.commons.types.descriptiontemplate.fielddata;
+
+import jakarta.xml.bind.annotation.XmlAccessType;
+import jakarta.xml.bind.annotation.XmlAccessorType;
+import jakarta.xml.bind.annotation.XmlAttribute;
+
+import java.util.UUID;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ReferenceTypeDataEntity extends BaseFieldDataEntity {
+    public static final String XmlElementName = "referenceTypeData";
+
+    @XmlAttribute(name = "multipleSelect")
+    private Boolean multipleSelect;
+
+    @XmlAttribute(name = "referenceTypeId")
+    private UUID referenceTypeId;
+    public Boolean getMultipleSelect() {
+        return multipleSelect;
+    }
+
+    public void setMultipleSelect(Boolean multipleSelect) {
+        this.multipleSelect = multipleSelect;
+    }
+
+    public UUID getReferenceTypeId() {
+        return referenceTypeId;
+    }
+
+    public void setReferenceTypeId(UUID referenceTypeId) {
+        this.referenceTypeId = referenceTypeId;
+    }
+}

dmp-backend/core/src/main/java/eu/eudat/commons/types/descriptiontemplate/importexport/FieldImportExport.java

@@ -40,7 +40,7 @@ public class FieldImportExport {
             @XmlElement(name = UploadDataImportExport.XmlElementName, type = UploadDataImportExport.class),
             @XmlElement(name = RadioBoxDataImportExport.XmlElementName, type = RadioBoxDataImportExport.class),
             @XmlElement(name = SelectDataImportExport.XmlElementName, type = SelectDataImportExport.class),
-            @XmlElement(name = ExternalSelectImportExport.XmlElementName, type = ExternalSelectImportExport.class),
+            @XmlElement(name = ReferenceTypeDataImportExport.XmlElementName, type = ReferenceTypeDataImportExport.class),
     })
     private BaseFieldDataImportExport data;
 

dmp-backend/core/src/main/java/eu/eudat/commons/types/descriptiontemplate/importexport/fielddata/ExternalSelectImportExport.java

@@ -1,174 +0,0 @@
-package eu.eudat.commons.types.descriptiontemplate.importexport.fielddata;
-
-import jakarta.xml.bind.annotation.*;
-
-import java.util.List;
-
-@XmlAccessorType(XmlAccessType.FIELD)
-public class ExternalSelectImportExport extends LabelAndMultiplicityDataImportExport {
-    public static final String XmlElementName = "externalSelectData";
-
-
-    @XmlAccessorType(XmlAccessType.FIELD)
-    public static class ExternalSelectAuthImportExport {
-        @XmlAttribute(name="url")
-        private String url;
-        @XmlAttribute(name="method")
-        private String method;
-        @XmlAttribute(name="body")
-        private String body;
-        @XmlAttribute(name="path")
-        private String path;
-        @XmlAttribute(name="type")
-        private String type;
-
-        public String getUrl() {
-            return url;
-        }
-
-        public void setUrl(String url) {
-            this.url = url;
-        }
-
-        public String getMethod() {
-            return method;
-        }
-
-        public void setMethod(String method) {
-            this.method = method;
-        }
-
-        public String getBody() {
-            return body;
-        }
-
-        public void setBody(String body) {
-            this.body = body;
-        }
-
-        public String getPath() {
-            return path;
-        }
-
-        public void setPath(String path) {
-            this.path = path;
-        }
-
-        public String getType() {
-            return type;
-        }
-
-        public void setType(String type) {
-            this.type = type;
-        }
-    }
-
-    @XmlAccessorType(XmlAccessType.FIELD)
-    public static class ExternalSelectSourceBindingImportExport {
-        @XmlAttribute(name="label")
-        private String label;
-        @XmlAttribute(name="value")
-        private String value;
-        @XmlAttribute(name="source")
-        private String source;
-
-        public String getLabel() {
-            return label;
-        }
-        public void setLabel(String label) {
-            this.label = label;
-        }
-
-        public String getValue() {
-            return value;
-        }
-        public void setValue(String value) {
-            this.value = value;
-        }
-
-        public String getSource() {
-            return source;
-        }
-        public void setSource(String source) {
-            this.source = source;
-        }
-    }
-
-    @XmlAccessorType(XmlAccessType.FIELD)
-    public static class ExternalSelectSourceImportExport {
-        @XmlAttribute(name="url")
-        private String url;
-        @XmlAttribute(name="method")
-        private String method;
-        @XmlAttribute(name="optionsRoot")
-        private String optionsRoot;
-        @XmlElement(name="sourceBinding")
-        private ExternalSelectSourceBindingImportExport sourceBinding;
-        @XmlAttribute(name="hasAuth")
-        private Boolean hasAuth;
-        @XmlElement(name="auth")
-        private ExternalSelectAuthImportExport auth;
-
-        public String getUrl() {
-            return url;
-        }
-
-        public void setUrl(String url) {
-            this.url = url;
-        }
-
-        public String getMethod() {
-            return method;
-        }
-
-        public void setMethod(String method) {
-            this.method = method;
-        }
-
-        public String getOptionsRoot() {
-            return optionsRoot;
-        }
-
-        public void setOptionsRoot(String optionsRoot) {
-            this.optionsRoot = optionsRoot;
-        }
-
-        public ExternalSelectSourceBindingImportExport getSourceBinding() {
-            return sourceBinding;
-        }
-
-        public void setSourceBinding(ExternalSelectSourceBindingImportExport sourceBinding) {
-            this.sourceBinding = sourceBinding;
-        }
-
-        public Boolean getHasAuth() {
-            return hasAuth;
-        }
-
-        public void setHasAuth(Boolean hasAuth) {
-            this.hasAuth = hasAuth;
-        }
-
-        public ExternalSelectAuthImportExport getAuth() {
-            return auth;
-        }
-
-        public void setAuth(ExternalSelectAuthImportExport auth) {
-            this.auth = auth;
-        }
-    }
-    @XmlElementWrapper(name = "sources")
-    @XmlElement(name = "source")
-    private List<ExternalSelectSourceImportExport> sources;
-
-    public List<ExternalSelectSourceImportExport> getSources() {
-        return sources;
-    }
-
-    public void setSources(List<ExternalSelectSourceImportExport> sources) {
-        this.sources = sources;
-    }
-    
-}
-
-

dmp-backend/core/src/main/java/eu/eudat/commons/types/descriptiontemplate/importexport/fielddata/LabelAndMultiplicityDataImportExport.java

@@ -20,3 +20,4 @@ public class LabelAndMultiplicityDataImportExport extends BaseFieldDataImportExp
     }
 
 }
+

dmp-backend/core/src/main/java/eu/eudat/commons/types/descriptiontemplate/importexport/fielddata/ReferenceTypeDataImportExport.java

@@ -0,0 +1,34 @@
+package eu.eudat.commons.types.descriptiontemplate.importexport.fielddata;
+
+import jakarta.xml.bind.annotation.XmlAccessType;
+import jakarta.xml.bind.annotation.XmlAccessorType;
+import jakarta.xml.bind.annotation.XmlAttribute;
+
+import java.util.UUID;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ReferenceTypeDataImportExport extends BaseFieldDataImportExport {
+    public static final String XmlElementName = "labelAndMultiplicityData";
+
+    @XmlAttribute(name = "multipleSelect")
+    private Boolean multipleSelect;
+
+    @XmlAttribute(name = "referenceTypeId")
+    private UUID referenceTypeId;
+
+    public Boolean getMultipleSelect() {
+        return multipleSelect;
+    }
+
+    public void setMultipleSelect(Boolean multipleSelect) {
+        this.multipleSelect = multipleSelect;
+    }
+
+    public UUID getReferenceTypeId() {
+        return referenceTypeId;
+    }
+
+    public void setReferenceTypeId(UUID referenceTypeId) {
+        this.referenceTypeId = referenceTypeId;
+    }
+}

dmp-backend/core/src/main/java/eu/eudat/commons/types/dmpblueprint/DefinitionEntity.java

@@ -2,7 +2,9 @@ package eu.eudat.commons.types.dmpblueprint;
 
 import jakarta.xml.bind.annotation.*;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.UUID;
 
 @XmlRootElement(name = "root")
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -17,4 +19,18 @@ public class DefinitionEntity {
 	public void setSections(List<SectionEntity> sections) {
 		this.sections = sections;
 	}
+
+	public List<FieldEntity> getAllField(){
+		List<FieldEntity> fieldEntities = new ArrayList<>();
+		if (this.getSections() != null){
+			for (SectionEntity sectionEntity: this.getSections()) {
+				fieldEntities.addAll(sectionEntity.getAllField());
+			}
+		}
+		return fieldEntities;
+	}
+
+	public List<FieldEntity> getFieldById(UUID id) {
+		return this.getAllField().stream().filter(x-> id.equals(x.getId())).toList();
+	}
 }

dmp-backend/core/src/main/java/eu/eudat/commons/types/dmpblueprint/ReferenceTypeFieldEntity.java

@@ -0,0 +1,22 @@
+package eu.eudat.commons.types.dmpblueprint;
+
+import jakarta.xml.bind.annotation.XmlAccessType;
+import jakarta.xml.bind.annotation.XmlAccessorType;
+import jakarta.xml.bind.annotation.XmlAttribute;
+
+import java.util.UUID;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ReferenceTypeFieldEntity extends FieldEntity {
+
+    @XmlAttribute(name="referenceTypeId")
+    private UUID referenceTypeId;
+
+    public UUID getReferenceTypeId() {
+        return referenceTypeId;
+    }
+
+    public void setReferenceTypeId(UUID referenceTypeId) {
+        this.referenceTypeId = referenceTypeId;
+    }
+}

dmp-backend/core/src/main/java/eu/eudat/commons/types/dmpblueprint/SectionEntity.java

@@ -3,6 +3,7 @@ package eu.eudat.commons.types.dmpblueprint;
 import eu.eudat.commons.enums.DmpBlueprintFieldCategory;
 import jakarta.xml.bind.annotation.*;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
 
@@ -25,6 +26,7 @@ public class SectionEntity {
     @XmlElements({
             @XmlElement(name = DmpBlueprintFieldCategory.Names.Extra, type = ExtraFieldEntity.class),
             @XmlElement(name = DmpBlueprintFieldCategory.Names.System, type = SystemFieldEntity.class),
+            @XmlElement(name = DmpBlueprintFieldCategory.Names.ReferenceType, type = ReferenceTypeFieldEntity.class),
     })
     private List<FieldEntity> fields;
     
@@ -84,4 +86,8 @@ public class SectionEntity {
         this.descriptionTemplates = descriptionTemplates;
     }
 
+    public List<FieldEntity> getAllField(){
+        return this.getFields() != null ? this.getFields() : new ArrayList<>();
+    }
+
 }

dmp-backend/core/src/main/java/eu/eudat/commons/types/dmpblueprint/SystemFieldEntity.java

@@ -5,6 +5,8 @@ import jakarta.xml.bind.annotation.XmlAccessType;
 import jakarta.xml.bind.annotation.XmlAccessorType;
 import jakarta.xml.bind.annotation.XmlAttribute;
 
+import java.util.UUID;
+
 @XmlAccessorType(XmlAccessType.FIELD)
 public class SystemFieldEntity extends FieldEntity {
 

dmp-backend/core/src/main/java/eu/eudat/commons/types/dmpblueprint/importexport/ReferenceTypeFieldImportExport.java

@@ -0,0 +1,84 @@
+package eu.eudat.commons.types.dmpblueprint.importexport;
+
+import jakarta.xml.bind.annotation.XmlAccessType;
+import jakarta.xml.bind.annotation.XmlAccessorType;
+import jakarta.xml.bind.annotation.XmlAttribute;
+
+import java.util.UUID;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ReferenceTypeFieldImportExport {
+
+    @XmlAttribute(name = "id")
+    private UUID id;
+    @XmlAttribute(name = "referenceTypeId")
+    private UUID referenceTypeId;
+    @XmlAttribute(name = "label")
+    private String label;
+    @XmlAttribute(name = "placeholder")
+    private String placeholder;
+    @XmlAttribute(name = "description")
+    private String description;
+    @XmlAttribute(name = "ordinal")
+    private int ordinal;
+    @XmlAttribute(name = "required")
+    private boolean required;
+
+    public UUID getId() {
+        return id;
+    }
+
+    public void setId(UUID id) {
+        this.id = id;
+    }
+
+    public UUID getReferenceTypeId() {
+        return referenceTypeId;
+    }
+
+    public void setReferenceTypeId(UUID referenceTypeId) {
+        this.referenceTypeId = referenceTypeId;
+    }
+
+    public String getLabel() {
+        return label;
+    }
+
+    public void setLabel(String label) {
+        this.label = label;
+    }
+
+    public String getPlaceholder() {
+        return placeholder;
+    }
+
+    public void setPlaceholder(String placeholder) {
+        this.placeholder = placeholder;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public int getOrdinal() {
+        return ordinal;
+    }
+
+    public void setOrdinal(int ordinal) {
+        this.ordinal = ordinal;
+    }
+
+    public boolean isRequired() {
+        return required;
+    }
+
+    public void setRequired(boolean required) {
+        this.required = required;
+    }
+
+
+}

dmp-backend/core/src/main/java/eu/eudat/commons/types/dmpblueprint/importexport/SectionImportExport.java

@@ -22,6 +22,9 @@ public class SectionImportExport {
     @XmlElementWrapper(name = "extraFields")
     @XmlElement(name = "extraField")
     private List<ExtraFieldImportExport> extraFields;
+    @XmlElementWrapper(name = "referenceFields")
+    @XmlElement(name = "referenceField")
+    private List<ReferenceTypeFieldImportExport> referenceFields;
     @XmlAttribute(name = "hasTemplates")
     private boolean hasTemplates;
     @XmlElementWrapper(name = "descriptionTemplates")
@@ -76,6 +79,14 @@ public class SectionImportExport {
         this.extraFields = extraFields;
     }
 
+    public List<ReferenceTypeFieldImportExport> getReferenceFields() {
+        return referenceFields;
+    }
+
+    public void setReferenceFields(List<ReferenceTypeFieldImportExport> referenceFields) {
+        this.referenceFields = referenceFields;
+    }
+
     public boolean isHasTemplates() {
         return hasTemplates;
     }

dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/AuthenticationConfigurationEntity.java

@@ -1,9 +1,10 @@
 package eu.eudat.commons.types.referencetype;
 
 import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
+import eu.eudat.service.remotefetcher.config.entities.AuthenticationConfiguration;
 import jakarta.xml.bind.annotation.XmlElement;
 
-public class AuthenticationConfigurationEntity {
+public class AuthenticationConfigurationEntity implements AuthenticationConfiguration {
 
     private Boolean enabled;
     private String authUrl;

dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/QueryConfigEntity.java

@@ -1,8 +1,9 @@
 package eu.eudat.commons.types.referencetype;
 
+import eu.eudat.service.remotefetcher.config.entities.QueryConfig;
 import jakarta.xml.bind.annotation.XmlElement;
 
-public class QueryConfigEntity {
+public class QueryConfigEntity implements QueryConfig {
 
     private String condition;
     private String separator;

dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceBaseConfigurationEntity.java

@@ -1,12 +1,13 @@
 package eu.eudat.commons.types.referencetype;
 
 import eu.eudat.commons.enums.ReferenceTypeSourceType;
+import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
 import jakarta.xml.bind.annotation.XmlElement;
 import jakarta.xml.bind.annotation.XmlElementWrapper;
 
 import java.util.List;
 
-public abstract class ReferenceTypeSourceBaseConfigurationEntity {
+public abstract class ReferenceTypeSourceBaseConfigurationEntity implements SourceBaseConfiguration {
 
     private String key;
 

dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceExternalApiConfigurationEntity.java

@@ -2,11 +2,12 @@ package eu.eudat.commons.types.referencetype;
 
 
 import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
+import eu.eudat.service.remotefetcher.config.entities.SourceExternalApiConfiguration;
 import jakarta.xml.bind.annotation.XmlElement;
 import jakarta.xml.bind.annotation.XmlElementWrapper;
 
 import java.util.List;
-public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity {
+public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceExternalApiConfiguration<ResultsConfigurationEntity, AuthenticationConfigurationEntity, QueryConfigEntity> {
 
     private String url;
     private ResultsConfigurationEntity results;

dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceStaticOptionConfigurationEntity.java

@@ -1,11 +1,12 @@
 package eu.eudat.commons.types.referencetype;
 
+import eu.eudat.service.remotefetcher.config.entities.SourceStaticOptionConfiguration;
 import jakarta.xml.bind.annotation.XmlElement;
 import jakarta.xml.bind.annotation.XmlElementWrapper;
 
 import java.util.List;
 
-public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity {
+public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceStaticOptionConfiguration<ReferenceTypeStaticOptionEntity> {
 
     List<ReferenceTypeStaticOptionEntity> options;
 

dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeStaticOptionEntity.java

@@ -1,8 +1,9 @@
 package eu.eudat.commons.types.referencetype;
 
+import eu.eudat.service.remotefetcher.config.entities.StaticOption;
 import jakarta.xml.bind.annotation.XmlElement;
 
-public class ReferenceTypeStaticOptionEntity {
+public class ReferenceTypeStaticOptionEntity implements StaticOption {
 
     private String code;
 

dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultFieldsMappingConfigurationEntity.java

@@ -1,8 +1,9 @@
 package eu.eudat.commons.types.referencetype;
 
+import eu.eudat.service.remotefetcher.config.entities.ResultFieldsMappingConfiguration;
 import jakarta.xml.bind.annotation.XmlElement;
 
-public class ResultFieldsMappingConfigurationEntity {
+public class ResultFieldsMappingConfigurationEntity implements ResultFieldsMappingConfiguration {
     private String code;
     private String responsePath;
 

dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultsConfigurationEntity.java

@@ -1,11 +1,12 @@
 package eu.eudat.commons.types.referencetype;
 
+import eu.eudat.service.remotefetcher.config.entities.ResultsConfiguration;
 import jakarta.xml.bind.annotation.XmlElement;
 import jakarta.xml.bind.annotation.XmlElementWrapper;
 
 import java.util.List;
 
-public class ResultsConfigurationEntity {
+public class ResultsConfigurationEntity implements ResultsConfiguration<ResultFieldsMappingConfigurationEntity> {
     private String resultsArrayPath;
     private List<ResultFieldsMappingConfigurationEntity> fieldsMapping;
 

dmp-backend/core/src/main/java/eu/eudat/data/ReferenceEntity.java

@@ -2,10 +2,8 @@ package eu.eudat.data;
 
 import eu.eudat.commons.enums.IsActive;
 import eu.eudat.commons.enums.ReferenceSourceType;
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.data.converters.enums.IsActiveConverter;
 import eu.eudat.data.converters.enums.ReferenceSourceTypeConverter;
-import eu.eudat.data.converters.enums.ReferenceTypeConverter;
 import eu.eudat.data.tenant.TenantScopedBaseEntity;
 import jakarta.persistence.*;
 
@@ -30,10 +28,9 @@ public class ReferenceEntity extends TenantScopedBaseEntity {
     public static final int _labelLength = 500;
 
     @Column(name = "type", nullable = false)
-    @Convert(converter = ReferenceTypeConverter.class)
+    private UUID typeId;
-    private ReferenceType type;
 
-    public static final String _type = "type";
+    public static final String _typeId = "typeId";
 
     @Column(name = "description")
     private String description;
@@ -109,12 +106,12 @@ public class ReferenceEntity extends TenantScopedBaseEntity {
         this.label = label;
     }
 
-    public ReferenceType getType() {
+    public UUID getTypeId() {
-        return type;
+        return typeId;
     }
 
-    public void setType(ReferenceType type) {
+    public void setTypeId(UUID typeId) {
-        this.type = type;
+        this.typeId = typeId;
     }
 
     public String getDescription() {

dmp-backend/core/src/main/java/eu/eudat/data/converters/DateToUTCConverter.java

@@ -1,48 +0,0 @@
-package eu.eudat.data.converters;
-
-import jakarta.persistence.AttributeConverter;
-import jakarta.persistence.Converter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.TimeZone;
-
-/**
- * Created by ikalyvas on 9/25/2018.
- */
-@Converter
-public class DateToUTCConverter implements AttributeConverter<Date, Date> {
-    private static final Logger logger = LoggerFactory.getLogger(DateToUTCConverter.class);
-
-    @Override
-    public Date convertToDatabaseColumn(Date attribute) {
-        if(attribute == null) return null;
-        DateFormat formatterIST = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-        formatterIST.setTimeZone(TimeZone.getTimeZone("UTC"));
-        try {
-            String date = formatterIST.format(attribute);
-            return formatterIST.parse(date);
-        } catch (ParseException e) {
-            logger.error(e.getMessage(), e);
-        }
-        return null;
-    }
-
-    @Override
-    public Date convertToEntityAttribute(Date dbData) {
-        if(dbData == null) return null;
-        DateFormat formatterIST = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-        formatterIST.setTimeZone(TimeZone.getTimeZone("UTC"));
-        try {
-            String date = formatterIST.format(dbData);
-            return formatterIST.parse(date);
-        } catch (ParseException e) {
-            logger.error(e.getMessage(), e);
-        }
-        return null;
-    }
-}

dmp-backend/core/src/main/java/eu/eudat/data/converters/enums/ReferenceTypeConverter.java

@@ -1,11 +0,0 @@
-package eu.eudat.data.converters.enums;
-
-import eu.eudat.commons.enums.ReferenceType;
-import jakarta.persistence.Converter;
-
-@Converter
-public class ReferenceTypeConverter extends DatabaseEnumConverter<ReferenceType, Short> {
-    public ReferenceType of(Short i) {
-        return ReferenceType.of(i);
-    }
-}

dmp-backend/core/src/main/java/eu/eudat/data/old/helpers/EntityBinder.java

@@ -1,14 +0,0 @@
-package eu.eudat.data.old.helpers;
-
-import jakarta.persistence.Tuple;
-import java.util.List;
-
-public class EntityBinder {
-	public static <T> T fromTuple(List<Tuple> tuple, String path) {
-		try {
-			return (T) tuple.get(0).get(path);
-		}catch (IllegalArgumentException illegalArgument){
-			return null;
-		}
-	}
-}

dmp-backend/core/src/main/java/eu/eudat/model/DashboardReferenceTypeStatistics.java

@@ -0,0 +1,27 @@
+package eu.eudat.model;
+
+public class DashboardReferenceTypeStatistics {
+
+    private long count;
+
+    public static final String _count = "count";
+
+    private PublicReferenceType referenceType;
+
+    public static final String _referenceType = "referenceType";
+    public long getCount() {
+        return count;
+    }
+
+    public void setCount(long count) {
+        this.count = count;
+    }
+
+    public PublicReferenceType getReferenceType() {
+        return referenceType;
+    }
+
+    public void setReferenceType(PublicReferenceType referenceType) {
+        this.referenceType = referenceType;
+    }
+}

dmp-backend/core/src/main/java/eu/eudat/model/DashboardStatistics.java

@@ -1,5 +1,8 @@
 package eu.eudat.model;
 
+import java.util.List;
+import java.util.UUID;
+
 public class DashboardStatistics {
 
     private long dmpCount;
@@ -8,15 +11,7 @@ public class DashboardStatistics {
 
     private long descriptionCount;
     
-    public static final String _descriptionCount = "descriptionCount";
+    private List<DashboardReferenceTypeStatistics> referenceTypeStatistics;
-
-    private long organizationCount;
-
-    public static final String _organizationCount = "organizationCount";
-
-    private long grantCount;
-
-    public static final String _grantCount = "grantCount";
 
     public long getDmpCount() {
         return dmpCount;
@@ -34,19 +29,11 @@ public class DashboardStatistics {
         this.descriptionCount = descriptionCount;
     }
 
-    public long getOrganizationCount() {
+    public List<DashboardReferenceTypeStatistics> getReferenceTypeStatistics() {
-        return organizationCount;
+        return referenceTypeStatistics;
     }
 
-    public void setOrganizationCount(long organizationCount) {
+    public void setReferenceTypeStatistics(List<DashboardReferenceTypeStatistics> referenceTypeStatistics) {
-        this.organizationCount = organizationCount;
+        this.referenceTypeStatistics = referenceTypeStatistics;
-    }
-
-    public long getGrantCount() {
-        return grantCount;
-    }
-
-    public void setGrantCount(long grantCount) {
-        this.grantCount = grantCount;
     }
 }

dmp-backend/core/src/main/java/eu/eudat/model/PublicReference.java

@@ -1,12 +1,5 @@
 package eu.eudat.model;
 
-import eu.eudat.commons.enums.IsActive;
-import eu.eudat.commons.enums.ReferenceSourceType;
-import eu.eudat.commons.enums.ReferenceType;
-import eu.eudat.model.referencedefinition.Definition;
-
-import java.time.Instant;
-import java.util.List;
 import java.util.UUID;
 
 public class PublicReference {
@@ -17,7 +10,7 @@ public class PublicReference {
     private String label;
     public static final String _label = "label";
 
-    private ReferenceType type;
+    private PublicReferenceType type;
     public static final String _type = "type";
 
     private String description;
@@ -42,11 +35,11 @@ public class PublicReference {
         this.label = label;
     }
 
-    public ReferenceType getType() {
+    public PublicReferenceType getType() {
         return type;
     }
 
-    public void setType(ReferenceType type) {
+    public void setType(PublicReferenceType type) {
         this.type = type;
     }
 
@@ -66,3 +59,5 @@ public class PublicReference {
         this.reference = reference;
     }
 }
+
+

dmp-backend/core/src/main/java/eu/eudat/model/PublicReferenceType.java

@@ -0,0 +1,30 @@
+package eu.eudat.model;
+
+import java.util.UUID;
+
+public class PublicReferenceType {
+
+    private UUID id;
+    public static final String _id = "id";
+
+
+    private String name;
+    public static final String _name = "name";
+
+
+    public UUID getId() {
+        return id;
+    }
+
+    public void setId(UUID id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+}

dmp-backend/core/src/main/java/eu/eudat/model/Reference.java

@@ -1,6 +1,5 @@
 package eu.eudat.model;
 
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.commons.enums.IsActive;
 import eu.eudat.commons.enums.ReferenceSourceType;
 import eu.eudat.model.referencedefinition.Definition;

dmp-backend/core/src/main/java/eu/eudat/model/builder/PublicReferenceBuilder.java

@@ -1,10 +1,20 @@
 package eu.eudat.model.builder;
 
 import eu.eudat.authorization.AuthorizationFlags;
+import eu.eudat.commons.enums.IsActive;
 import eu.eudat.convention.ConventionService;
+import eu.eudat.data.DmpReferenceEntity;
 import eu.eudat.data.ReferenceEntity;
+import eu.eudat.model.PublicDmp;
+import eu.eudat.model.PublicDmpUser;
 import eu.eudat.model.PublicReference;
+import eu.eudat.model.PublicReferenceType;
+import eu.eudat.query.DmpQuery;
+import eu.eudat.query.ReferenceTypeQuery;
+import gr.cite.tools.data.builder.BuilderFactory;
+import gr.cite.tools.data.query.QueryFactory;
 import gr.cite.tools.exception.MyApplicationException;
+import gr.cite.tools.fieldset.BaseFieldSet;
 import gr.cite.tools.fieldset.FieldSet;
 import gr.cite.tools.logging.DataLogEntry;
 import gr.cite.tools.logging.LoggerService;
@@ -15,17 +25,22 @@ import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Component;
 
 import java.util.*;
+import java.util.stream.Collectors;
 
 @Component
 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
 public class PublicReferenceBuilder extends BaseBuilder<PublicReference, ReferenceEntity>{
+    private final BuilderFactory builderFactory;
 
+    private final QueryFactory queryFactory;
     private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
 
     @Autowired
     public PublicReferenceBuilder(
-            ConventionService conventionService) {
+		    ConventionService conventionService, BuilderFactory builderFactory, QueryFactory queryFactory) {
         super(conventionService, new LoggerService(LoggerFactory.getLogger(PublicReferenceBuilder.class)));
+	    this.builderFactory = builderFactory;
+	    this.queryFactory = queryFactory;
     }
 
     public PublicReferenceBuilder authorize(EnumSet<AuthorizationFlags> values) {
@@ -40,6 +55,10 @@ public class PublicReferenceBuilder extends BaseBuilder<PublicReference, Referen
         if (fields == null || data == null || fields.isEmpty())
             return new ArrayList<>();
 
+
+        FieldSet typeFields = fields.extractPrefixed(this.asPrefix(PublicReference._type));
+        Map<UUID, PublicReferenceType> typeItemsMap = this.collectReferenceTypes(typeFields, data);
+        
         List<PublicReference> models = new ArrayList<>();
         for (ReferenceEntity d : data) {
             PublicReference m = new PublicReference();
@@ -47,10 +66,39 @@ public class PublicReferenceBuilder extends BaseBuilder<PublicReference, Referen
             if (fields.hasField(this.asIndexer(PublicReference._label))) m.setLabel(d.getLabel());
             if (fields.hasField(this.asIndexer(PublicReference._reference))) m.setReference(d.getReference());
             if (fields.hasField(this.asIndexer(PublicReference._description))) m.setDescription(d.getDescription());
-            if (fields.hasField(this.asIndexer(PublicReference._type))) m.setType(d.getType());
+            if (!typeFields.isEmpty() && typeItemsMap != null && typeItemsMap.containsKey(d.getTypeId())) m.setType(typeItemsMap.get(d.getTypeId()));
             models.add(m);
         }
         this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
         return models;
     }
+
+
+
+    private Map<UUID, PublicReferenceType> collectReferenceTypes(FieldSet fields, List<ReferenceEntity> data) throws MyApplicationException {
+        if (fields.isEmpty() || data.isEmpty())
+            return null;
+        this.logger.debug("checking related - {}", PublicReferenceType.class.getSimpleName());
+
+        Map<UUID, PublicReferenceType> itemMap;
+        if (!fields.hasOtherField(this.asIndexer(PublicReferenceType._id))) {
+            itemMap = this.asEmpty(
+                    data.stream().map(ReferenceEntity::getTypeId).distinct().collect(Collectors.toList()),
+                    x -> {
+                        PublicReferenceType item = new PublicReferenceType();
+                        item.setId(x);
+                        return item;
+                    },
+                    PublicReferenceType::getId);
+        } else {
+            FieldSet clone = new BaseFieldSet(fields.getFields()).ensure(PublicReferenceType._id);
+            ReferenceTypeQuery q = this.queryFactory.query(ReferenceTypeQuery.class).authorize(this.authorize).isActive(IsActive.Active).ids(data.stream().map(ReferenceEntity::getTypeId).distinct().collect(Collectors.toList()));
+            itemMap = this.builderFactory.builder(PublicReferenceTypeBuilder.class).authorize(this.authorize).asForeignKey(q, clone, PublicReferenceType::getId);
+        }
+        if (!fields.hasField(PublicReferenceType._id)) {
+            itemMap.values().stream().filter(Objects::nonNull).peek(x -> x.setId(null)).collect(Collectors.toList());
+        }
+
+        return itemMap;
+    }
 }

dmp-backend/core/src/main/java/eu/eudat/model/builder/PublicReferenceTypeBuilder.java

@@ -1,11 +1,9 @@
-package eu.eudat.model.builder.referencesearch;
+package eu.eudat.model.builder;
 
 import eu.eudat.authorization.AuthorizationFlags;
-import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
 import eu.eudat.convention.ConventionService;
-import eu.eudat.model.builder.BaseBuilder;
+import eu.eudat.data.ReferenceTypeEntity;
-import eu.eudat.model.referencedefinition.Definition;
+import eu.eudat.model.PublicReferenceType;
-import gr.cite.tools.data.builder.BuilderFactory;
 import gr.cite.tools.exception.MyApplicationException;
 import gr.cite.tools.fieldset.FieldSet;
 import gr.cite.tools.logging.DataLogEntry;
@@ -20,38 +18,35 @@ import java.util.*;
 
 @Component
 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
-public class DefinitionSearchBuilder extends BaseBuilder<Definition, Map<String, String>> {
+public class PublicReferenceTypeBuilder extends BaseBuilder<PublicReferenceType, ReferenceTypeEntity>{
 
-    private final BuilderFactory builderFactory;
-    private final ReferenceTypeProperties properties;
     private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
 
     @Autowired
-    public DefinitionSearchBuilder(
+    public PublicReferenceTypeBuilder(
-            ConventionService conventionService, BuilderFactory builderFactory, ReferenceTypeProperties properties) {
+            ConventionService conventionService) {
-        super(conventionService, new LoggerService(LoggerFactory.getLogger(DefinitionSearchBuilder.class)));
+        super(conventionService, new LoggerService(LoggerFactory.getLogger(PublicReferenceTypeBuilder.class)));
-        this.builderFactory = builderFactory;
-        this.properties = properties;
     }
 
-    public DefinitionSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
+    public PublicReferenceTypeBuilder authorize(EnumSet<AuthorizationFlags> values) {
         this.authorize = values;
         return this;
     }
 
     @Override
-    public List<Definition> build(FieldSet fields, List<Map<String, String>> data) throws MyApplicationException {
+    public List<PublicReferenceType> build(FieldSet fields, List<ReferenceTypeEntity> data) throws MyApplicationException {
         this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
         this.logger.trace(new DataLogEntry("requested fields", fields));
-        if (data == null)
+        if (fields == null || data == null || fields.isEmpty())
             return new ArrayList<>();
 
-        List<Definition> models = new ArrayList<>();
+        List<PublicReferenceType> models = new ArrayList<>();
-        //for (Map<String, String> d : data) {
+        for (ReferenceTypeEntity d : data) {
-        Definition m = new Definition();
+            PublicReferenceType m = new PublicReferenceType();
-        m.setFields(this.builderFactory.builder(FieldSearchBuilder.class).authorize(this.authorize).build(null, data));
+            if (fields.hasField(this.asIndexer(PublicReferenceType._id))) m.setId(d.getId());
+            if (fields.hasField(this.asIndexer(PublicReferenceType._name))) m.setName(d.getName());
             models.add(m);
-        //}
+        }
         this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
         return models;
     }

dmp-backend/core/src/main/java/eu/eudat/model/builder/ReferenceBuilder.java

@@ -8,9 +8,11 @@ import eu.eudat.data.DescriptionEntity;
 import eu.eudat.data.ReferenceEntity;
 import eu.eudat.model.DmpReference;
 import eu.eudat.model.Reference;
+import eu.eudat.model.ReferenceType;
 import eu.eudat.model.User;
 import eu.eudat.model.builder.referencedefinition.DefinitionBuilder;
 import eu.eudat.query.DmpReferenceQuery;
+import eu.eudat.query.ReferenceTypeQuery;
 import eu.eudat.query.UserQuery;
 import gr.cite.tools.data.builder.BuilderFactory;
 import gr.cite.tools.data.query.QueryFactory;
@@ -68,6 +70,9 @@ public class ReferenceBuilder extends BaseBuilder<Reference, ReferenceEntity>{
         FieldSet userFields = fields.extractPrefixed(this.asPrefix(Reference._createdBy));
         Map<UUID, User> userItemsMap = this.collectUsers(userFields, data);
 
+        FieldSet typeFields = fields.extractPrefixed(this.asPrefix(Reference._type));
+        Map<UUID, ReferenceType> typeItemsMap = this.collectReferenceTypes(typeFields, data);
+
         List<Reference> models = new ArrayList<>();
         for (ReferenceEntity d : data) {
             Reference m = new Reference();
@@ -86,7 +91,7 @@ public class ReferenceBuilder extends BaseBuilder<Reference, ReferenceEntity>{
             if (fields.hasField(this.asIndexer(Reference._description))) m.setDescription(d.getDescription());
             if (fields.hasField(this.asIndexer(Reference._source))) m.setSource(d.getSource());
             if (fields.hasField(this.asIndexer(Reference._sourceType))) m.setSourceType(d.getSourceType());
-            if (fields.hasField(this.asIndexer(Reference._type))) m.setType(d.getType());
+            if (!typeFields.isEmpty() && typeItemsMap != null && typeItemsMap.containsKey(d.getTypeId())) m.setType(typeItemsMap.get(d.getTypeId()));
             if (dmpReferenceMap != null && !dmpReferenceMap.isEmpty() && dmpReferenceMap.containsKey(d.getId())) m.setDmpReferences(dmpReferenceMap.get(d.getId()));
             if (!userFields.isEmpty() && userItemsMap != null && userItemsMap.containsKey(d.getCreatedById())) m.setCreatedBy(userItemsMap.get(d.getCreatedById()));
             models.add(m);
@@ -95,6 +100,36 @@ public class ReferenceBuilder extends BaseBuilder<Reference, ReferenceEntity>{
         return models;
     }
 
+    private Map<UUID, ReferenceType> collectReferenceTypes(FieldSet fields, List<ReferenceEntity> data) throws MyApplicationException {
+        if (fields.isEmpty() || data.isEmpty())
+            return null;
+        this.logger.debug("checking related - {}", ReferenceType.class.getSimpleName());
+
+        Map<UUID, ReferenceType> itemMap;
+        if (!fields.hasOtherField(this.asIndexer(ReferenceType._id))) {
+            itemMap = this.asEmpty(
+                    data.stream().map(ReferenceEntity::getTypeId).distinct().collect(Collectors.toList()),
+                    x -> {
+                        ReferenceType item = new ReferenceType();
+                        item.setId(x);
+                        return item;
+                    },
+                    ReferenceType::getId);
+        } else {
+            FieldSet clone = new BaseFieldSet(fields.getFields()).ensure(ReferenceType._id);
+            ReferenceTypeQuery q = this.queryFactory.query(ReferenceTypeQuery.class).authorize(this.authorize).ids(data.stream().map(ReferenceEntity::getTypeId).distinct().collect(Collectors.toList()));
+            itemMap = this.builderFactory.builder(ReferenceTypeBuilder.class).authorize(this.authorize).asForeignKey(q, clone, ReferenceType::getId);
+        }
+        if (!fields.hasField(ReferenceType._id)) {
+            itemMap.forEach((id, item) -> {
+                if (item != null)
+                    item.setId(null);
+            });
+        }
+
+        return itemMap;
+    }
+
     private Map<UUID, User> collectUsers(FieldSet fields, List<ReferenceEntity> data) throws MyApplicationException {
         if (fields.isEmpty() || data.isEmpty())
             return null;

dmp-backend/core/src/main/java/eu/eudat/model/builder/deposit/DescriptionFieldDepositBuilder.java

@@ -80,11 +80,9 @@ public class DescriptionFieldDepositBuilder extends BaseDepositBuilder<Descripti
                         }
                         
                         switch (field.getData().getFieldType()) {
-                            case EXTERNAL_SELECT -> m.setFieldType(FieldType.AUTO_COMPLETE);
                             case SELECT -> m.setFieldType(FieldType.WORD_LIST);
                             case BOOLEAN_DECISION -> m.setFieldType(FieldType.BOOLEAN_DECISION);
                             case RADIO_BOX -> m.setFieldType(FieldType.RADIO_BOX);
-                            case INTERNAL_DMP_ENTRIES_RESEARCHERS -> m.setFieldType(FieldType.INTERNAL_DMP_ENTRIES_RESEARCHERS);
                             case INTERNAL_DMP_ENTRIES_DMPS -> m.setFieldType(FieldType.INTERNAL_DMP_ENTRIES_DMPS);
                             case INTERNAL_DMP_ENTRIES_DATASETS -> m.setFieldType(FieldType.INTERNAL_DMP_ENTRIES_DATASETS);
                             case CHECK_BOX -> m.setFieldType(FieldType.CHECK_BOX);
@@ -94,20 +92,11 @@ public class DescriptionFieldDepositBuilder extends BaseDepositBuilder<Descripti
                             case UPLOAD -> m.setFieldType(FieldType.UPLOAD);
                             case DATE_PICKER -> m.setFieldType(FieldType.DATE_PICKER);
                             case EXTERNAL_DATASETS -> m.setFieldType(FieldType.EXTERNAL_DATASETS);
-                            case DATA_REPOSITORIES -> m.setFieldType(FieldType.DATA_REPOSITORIES);
+//                            case REFERENCE_TYPES -> m.setFieldType(FieldType.REFERENCE_TYPES); //TODO new reference logic
-                            case JOURNAL_REPOSITORIES -> m.setFieldType(FieldType.JOURNAL_REPOSITORIES);
-                            case PUB_REPOSITORIES -> m.setFieldType(FieldType.PUB_REPOSITORIES);
-                            case LICENSES -> m.setFieldType(FieldType.LICENSES);
-                            case PUBLICATIONS -> m.setFieldType(FieldType.PUBLICATIONS);
-                            case REGISTRIES -> m.setFieldType(FieldType.REGISTRIES);
-                            case SERVICES -> m.setFieldType(FieldType.SERVICES);
                             case TAGS -> m.setFieldType(FieldType.TAGS);
-                            case RESEARCHERS -> m.setFieldType(FieldType.RESEARCHERS);
-                            case ORGANIZATIONS -> m.setFieldType(FieldType.ORGANIZATIONS);
                             case DATASET_IDENTIFIER -> m.setFieldType(FieldType.DATASET_IDENTIFIER);
                             case CURRENCY -> m.setFieldType(FieldType.CURRENCY);
                             case VALIDATION -> m.setFieldType(FieldType.VALIDATION);
-                            case TAXONOMIES ->  m.setFieldType(FieldType.TAXONOMIES);
                             default -> throw new MyApplicationException("unrecognized type " + field.getData().getFieldType());
                         }
                     }

dmp-backend/core/src/main/java/eu/eudat/model/builder/deposit/ReferenceDepositBuilder.java

@@ -6,7 +6,6 @@ import eu.eudat.commons.types.reference.DefinitionEntity;
 import eu.eudat.convention.ConventionService;
 import eu.eudat.data.ReferenceEntity;
 import eu.eudat.depositinterface.enums.ReferenceSourceType;
-import eu.eudat.depositinterface.enums.ReferenceType;
 import eu.eudat.depositinterface.models.ReferenceDepositModel;
 import gr.cite.tools.data.builder.BuilderFactory;
 import gr.cite.tools.exception.MyApplicationException;
@@ -67,23 +66,7 @@ public class ReferenceDepositBuilder extends BaseDepositBuilder<ReferenceDeposit
                 case External -> m.setSourceType(ReferenceSourceType.External);
                 default -> throw new MyApplicationException("unrecognized type " + d.getSourceType().getValue());
             }
-            switch (d.getType()){
+//            m.setType(d.getTypeId()); //TODO new reference logic
-                case Taxonomies -> m.setType(ReferenceType.Taxonomies);
-                case Licenses -> m.setType(ReferenceType.Licenses);
-                case Publications -> m.setType(ReferenceType.Publications);
-                case Journals -> m.setType(ReferenceType.Journals);
-                case PubRepositories -> m.setType(ReferenceType.PubRepositories);
-                case DataRepositories -> m.setType(ReferenceType.DataRepositories);
-                case Registries -> m.setType(ReferenceType.Registries);
-                case Services -> m.setType(ReferenceType.Services);
-                case Project -> m.setType(ReferenceType.Project);
-                case Funder -> m.setType(ReferenceType.Funder);
-                case Datasets -> m.setType(ReferenceType.Datasets);
-                case Organizations -> m.setType(ReferenceType.Organizations);
-                case Grants -> m.setType(ReferenceType.Grants);
-                case Researcher -> m.setType(ReferenceType.Researcher);
-                default -> throw new MyApplicationException("unrecognized type " + d.getType().getValue());
-            }
             models.add(new DepositBuilderItemResponse<>(m, d));
         }
         this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));

dmp-backend/core/src/main/java/eu/eudat/model/builder/descriptiontemplatedefinition/fielddata/ExternalSelectDataBuilder.java

@@ -1,163 +0,0 @@
-package eu.eudat.model.builder.descriptiontemplatedefinition.fielddata;
-
-import eu.eudat.authorization.AuthorizationFlags;
-import eu.eudat.commons.types.descriptiontemplate.fielddata.ExternalSelectDataEntity;
-import eu.eudat.convention.ConventionService;
-import eu.eudat.model.builder.BaseBuilder;
-import eu.eudat.model.descriptiontemplatedefinition.fielddata.ExternalSelectData;
-import gr.cite.tools.data.builder.BuilderFactory;
-import gr.cite.tools.exception.MyApplicationException;
-import gr.cite.tools.fieldset.FieldSet;
-import gr.cite.tools.logging.DataLogEntry;
-import gr.cite.tools.logging.LoggerService;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.config.ConfigurableBeanFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import java.util.*;
-
-@Component
-@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
-public class ExternalSelectDataBuilder extends BaseFieldDataBuilder<ExternalSelectData, ExternalSelectDataEntity> {
-	private final BuilderFactory builderFactory;
-	@Autowired
-	public ExternalSelectDataBuilder(ConventionService conventionService, BuilderFactory builderFactory) {
-		super(conventionService, new LoggerService(LoggerFactory.getLogger(ExternalSelectDataBuilder.class)));
-		this.builderFactory = builderFactory;
-	}
-
-	@Override
-	protected ExternalSelectData getInstance() {
-		return new ExternalSelectData();
-	}
-
-	@Override
-	protected void buildChild(FieldSet fields, ExternalSelectDataEntity d, ExternalSelectData m) {
-		FieldSet sourcesFields = fields.extractPrefixed(this.asPrefix(ExternalSelectData._sources));
-		if (fields.hasField(this.asIndexer(ExternalSelectData._multipleSelect))) m.setMultipleSelect(d.getMultipleSelect());
-		if (!sourcesFields.isEmpty() && d.getSources() != null) m.setSources(this.builderFactory.builder(ExternalSelectSourceBuilder.class).authorize(this.authorize).build(sourcesFields, d.getSources()));
-
-	}
-
-	@Component
-	@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
-	public static class ExternalSelectSourceBindingBuilder extends BaseBuilder<ExternalSelectData.ExternalSelectSourceBinding, ExternalSelectDataEntity.ExternalSelectSourceBindingEntity> {
-
-		private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
-
-		@Autowired
-		public ExternalSelectSourceBindingBuilder(
-				ConventionService conventionService) {
-			super(conventionService, new LoggerService(LoggerFactory.getLogger(SelectDataBuilder.SelectOptionBuilder.class)));
-		}
-
-		public ExternalSelectSourceBindingBuilder authorize(EnumSet<AuthorizationFlags> values) {
-			this.authorize = values;
-			return this;
-		}
-
-		@Override
-		public List<ExternalSelectData.ExternalSelectSourceBinding> build(FieldSet fields, List<ExternalSelectDataEntity.ExternalSelectSourceBindingEntity> data) throws MyApplicationException {
-			this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
-			this.logger.trace(new DataLogEntry("requested fields", fields));
-			if (fields == null || data == null || fields.isEmpty())
-				return new ArrayList<>();
-
-			List<ExternalSelectData.ExternalSelectSourceBinding> models = new ArrayList<>();
-			for (ExternalSelectDataEntity.ExternalSelectSourceBindingEntity d : data) {
-				ExternalSelectData.ExternalSelectSourceBinding m = new ExternalSelectData.ExternalSelectSourceBinding();
-				if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectSourceBinding._label))) m.setLabel(d.getLabel());
-				if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectSourceBinding._value))) m.setValue(d.getValue());
-				if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectSourceBinding._source))) m.setSource(d.getSource());
-				models.add(m);
-			}
-			this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
-			return models;
-		}
-	}
-
-	@Component
-	@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
-	public static class ExternalSelectSourceBuilder extends BaseBuilder<ExternalSelectData.ExternalSelectSource, ExternalSelectDataEntity.ExternalSelectSourceEntity> {
-	
-	    private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
-	    private final BuilderFactory builderFactory;
-	
-	    @Autowired
-	    public ExternalSelectSourceBuilder(
-	            ConventionService conventionService, BuilderFactory builderFactory) {
-	        super(conventionService, new LoggerService(LoggerFactory.getLogger(ExternalSelectSourceBuilder.class)));
-	        this.builderFactory = builderFactory;
-	    }
-	
-	    public ExternalSelectSourceBuilder authorize(EnumSet<AuthorizationFlags> values) {
-	        this.authorize = values;
-	        return this;
-	    }
-	
-	    @Override
-	    public List<ExternalSelectData.ExternalSelectSource> build(FieldSet fields, List<ExternalSelectDataEntity.ExternalSelectSourceEntity> data) throws MyApplicationException {
-	        this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
-	        this.logger.trace(new DataLogEntry("requested fields", fields));
-	        if (fields == null || data == null || fields.isEmpty())
-	            return new ArrayList<>();
-	
-	        FieldSet sourceBindingFields = fields.extractPrefixed(this.asPrefix(ExternalSelectData.ExternalSelectSource._sourceBinding));
-	        FieldSet authFields = fields.extractPrefixed(this.asPrefix(ExternalSelectData.ExternalSelectSource._auth));
-	        List<ExternalSelectData.ExternalSelectSource> models = new ArrayList<>();
-	        for (ExternalSelectDataEntity.ExternalSelectSourceEntity d : data) {
-	            ExternalSelectData.ExternalSelectSource m = new ExternalSelectData.ExternalSelectSource();
-	            if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectSource._url))) m.setUrl(d.getUrl());
-	            if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectSource._optionsRoot))) m.setOptionsRoot(d.getOptionsRoot());
-	            if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectSource._hasAuth))) m.setHasAuth(d.getHasAuth());
-	            if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectSource._method))) m.setMethod(d.getMethod());
-	            if (!sourceBindingFields.isEmpty() && d.getSourceBinding() != null) m.setSourceBinding(this.builderFactory.builder(ExternalSelectSourceBindingBuilder.class).authorize(this.authorize).build(sourceBindingFields, d.getSourceBinding()));
-	            if (!authFields.isEmpty() && d.getAuth() != null) m.setAuth(this.builderFactory.builder(ExternalSelectAuthDataBuilder.class).authorize(this.authorize).build(authFields, d.getAuth()));
-	            models.add(m);
-	        }
-	        this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
-	        return models;
-	    }
-	}
-
-	@Component
-	@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
-	public static class ExternalSelectAuthDataBuilder extends BaseBuilder<ExternalSelectData.ExternalSelectAuthData, ExternalSelectDataEntity.ExternalSelectAuthDataEntity> {
-	
-	    private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
-	
-	    @Autowired
-	    public ExternalSelectAuthDataBuilder(
-	            ConventionService conventionService) {
-	        super(conventionService, new LoggerService(LoggerFactory.getLogger(ExternalSelectAuthDataBuilder.class)));
-	    }
-	
-	    public ExternalSelectAuthDataBuilder authorize(EnumSet<AuthorizationFlags> values) {
-	        this.authorize = values;
-	        return this;
-	    }
-	
-	    @Override
-	    public List<ExternalSelectData.ExternalSelectAuthData> build(FieldSet fields, List<ExternalSelectDataEntity.ExternalSelectAuthDataEntity> data) throws MyApplicationException {
-	        this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
-	        this.logger.trace(new DataLogEntry("requested fields", fields));
-	        if (fields == null || data == null || fields.isEmpty())
-	            return new ArrayList<>();
-	
-	        List<ExternalSelectData.ExternalSelectAuthData> models = new ArrayList<>();
-	        for (ExternalSelectDataEntity.ExternalSelectAuthDataEntity d : data) {
-	            ExternalSelectData.ExternalSelectAuthData m = new ExternalSelectData.ExternalSelectAuthData();
-	            if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectAuthData._url))) m.setUrl(d.getUrl());
-	            if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectAuthData._method))) m.setMethod(d.getMethod());
-	            if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectAuthData._body))) m.setBody(d.getBody());
-	            if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectAuthData._path))) m.setPath(d.getPath());
-	            if (fields.hasField(this.asIndexer(ExternalSelectData.ExternalSelectAuthData._type))) m.setType(d.getType());
-	            models.add(m);
-	        }
-	        this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
-	        return models;
-	    }
-	}
-}

dmp-backend/core/src/main/java/eu/eudat/model/builder/descriptiontemplatedefinition/fielddata/ReferenceTypeDataBuilder.java

@@ -0,0 +1,31 @@
+package eu.eudat.model.builder.descriptiontemplatedefinition.fielddata;
+
+import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity;
+import eu.eudat.convention.ConventionService;
+import eu.eudat.model.descriptiontemplatedefinition.fielddata.ReferenceTypeData;
+import gr.cite.tools.fieldset.FieldSet;
+import gr.cite.tools.logging.LoggerService;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+@Component
+@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+public class ReferenceTypeDataBuilder extends BaseFieldDataBuilder<ReferenceTypeData, ReferenceTypeDataEntity> {
+
+	@Autowired
+	public ReferenceTypeDataBuilder(ConventionService conventionService) {
+		super(conventionService, new LoggerService(LoggerFactory.getLogger(ReferenceTypeDataBuilder.class)));
+	}
+
+	protected ReferenceTypeData getInstance() {
+		return new ReferenceTypeData();
+	}
+	@Override
+	protected void buildChild(FieldSet fields, ReferenceTypeDataEntity d, ReferenceTypeData m) {
+		if (fields.hasField(this.asIndexer(ReferenceTypeData._multipleSelect))) m.setMultipleSelect(d.getMultipleSelect());
+		if (fields.hasField(this.asIndexer(ReferenceTypeData._referenceTypeId))) m.setReferenceTypeId(d.getReferenceTypeId());
+	}
+}

dmp-backend/core/src/main/java/eu/eudat/model/builder/dmpblueprintdefinition/FieldBuilder.java

@@ -1,19 +1,14 @@
 package eu.eudat.model.builder.dmpblueprintdefinition;
 
 import eu.eudat.authorization.AuthorizationFlags;
-import eu.eudat.commons.types.dmpblueprint.DescriptionTemplateEntity;
-import eu.eudat.commons.types.dmpblueprint.ExtraFieldEntity;
 import eu.eudat.commons.types.dmpblueprint.FieldEntity;
 import eu.eudat.convention.ConventionService;
 import eu.eudat.model.builder.BaseBuilder;
-import eu.eudat.model.dmpblueprintdefinition.DescriptionTemplate;
-import eu.eudat.model.dmpblueprintdefinition.ExtraField;
 import eu.eudat.model.dmpblueprintdefinition.Field;
 import gr.cite.tools.exception.MyApplicationException;
 import gr.cite.tools.fieldset.FieldSet;
 import gr.cite.tools.logging.DataLogEntry;
 import gr.cite.tools.logging.LoggerService;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.config.ConfigurableBeanFactory;
 import org.springframework.context.annotation.Scope;

dmp-backend/core/src/main/java/eu/eudat/model/builder/dmpblueprintdefinition/ReferenceFieldBuilder.java

@@ -0,0 +1,47 @@
+package eu.eudat.model.builder.dmpblueprintdefinition;
+
+import eu.eudat.commons.types.dmpblueprint.ReferenceTypeFieldEntity;
+import eu.eudat.convention.ConventionService;
+import eu.eudat.model.Dmp;
+import eu.eudat.model.PublicReferenceType;
+import eu.eudat.model.builder.PublicReferenceTypeBuilder;
+import eu.eudat.model.builder.ReferenceTypeBuilder;
+import eu.eudat.model.dmpblueprintdefinition.ReferenceTypeField;
+import eu.eudat.query.ReferenceTypeQuery;
+import gr.cite.tools.data.builder.BuilderFactory;
+import gr.cite.tools.data.query.QueryFactory;
+import gr.cite.tools.fieldset.BaseFieldSet;
+import gr.cite.tools.fieldset.FieldSet;
+import gr.cite.tools.logging.LoggerService;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+@Component
+@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+public class ReferenceFieldBuilder extends FieldBuilder<ReferenceTypeField, ReferenceTypeFieldEntity> {
+	private final QueryFactory queryFactory;
+
+	private final BuilderFactory builderFactory;
+
+	@Autowired
+	public ReferenceFieldBuilder(
+			ConventionService conventionService, QueryFactory queryFactory, BuilderFactory builderFactory) {
+		super(conventionService, new LoggerService(LoggerFactory.getLogger(ReferenceFieldBuilder.class)));
+		this.queryFactory = queryFactory;
+		this.builderFactory = builderFactory;
+	}
+
+	protected ReferenceTypeField getInstance() {
+		return new ReferenceTypeField();
+	}
+
+	protected ReferenceTypeField buildChild(FieldSet fields, ReferenceTypeFieldEntity data, ReferenceTypeField model) {
+		FieldSet referenceTypeFields = fields.extractPrefixed(this.asPrefix(ReferenceTypeField._referenceType));
+		
+		if (data.getReferenceTypeId() != null && !referenceTypeFields.isEmpty() ) model.setReferenceType(this.builderFactory.builder(ReferenceTypeBuilder.class).build(referenceTypeFields, this.queryFactory.query(ReferenceTypeQuery.class).ids(data.getReferenceTypeId()).first())); //TODO: Optimize
+		return model;
+	}
+}

dmp-backend/core/src/main/java/eu/eudat/model/builder/dmpblueprintdefinition/SectionBuilder.java

@@ -2,10 +2,7 @@ package eu.eudat.model.builder.dmpblueprintdefinition;
 
 import eu.eudat.authorization.AuthorizationFlags;
 import eu.eudat.commons.enums.DmpBlueprintFieldCategory;
-import eu.eudat.commons.types.dmpblueprint.ExtraFieldEntity;
+import eu.eudat.commons.types.dmpblueprint.*;
-import eu.eudat.commons.types.dmpblueprint.FieldEntity;
-import eu.eudat.commons.types.dmpblueprint.SectionEntity;
-import eu.eudat.commons.types.dmpblueprint.SystemFieldEntity;
 import eu.eudat.convention.ConventionService;
 import eu.eudat.model.builder.BaseBuilder;
 import eu.eudat.model.dmpblueprintdefinition.Section;
@@ -21,7 +18,6 @@ import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Component;
 
 import java.util.*;
-import java.util.stream.Collectors;
 
 @Component("dmpblueprintdefinitionsectionbuilder")
 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
@@ -63,11 +59,13 @@ public class SectionBuilder extends BaseBuilder<Section, SectionEntity> {
             if (fields.hasField(this.asIndexer(Section._hasTemplates))) m.setHasTemplates(d.getHasTemplates());
             if (!descriptionTemplatesFields.isEmpty() && d.getDescriptionTemplates() != null) m.setDescriptionTemplates(this.builderFactory.builder(DescriptionTemplateBuilder.class).authorize(this.authorize).build(descriptionTemplatesFields, d.getDescriptionTemplates()));
             if (!fieldsFields.isEmpty() && d.getFields() != null) {
-                List<SystemFieldEntity> systemFieldEntities = d.getFields().stream().filter(x-> DmpBlueprintFieldCategory.System.equals(x.getCategory())).map(x-> (SystemFieldEntity)x).toList();
-                List<ExtraFieldEntity> extraFieldEntities = d.getFields().stream().filter(x-> DmpBlueprintFieldCategory.Extra.equals(x.getCategory())).map(x-> (ExtraFieldEntity)x).toList();
                 m.setFields(new ArrayList<>());
+                List<SystemFieldEntity> systemFieldEntities = d.getFields().stream().filter(x-> DmpBlueprintFieldCategory.System.equals(x.getCategory())).map(x-> (SystemFieldEntity)x).toList();
                 m.getFields().addAll(this.builderFactory.builder(SystemFieldBuilder.class).authorize(this.authorize).build(fieldsFields, systemFieldEntities));
+                List<ExtraFieldEntity> extraFieldEntities = d.getFields().stream().filter(x-> DmpBlueprintFieldCategory.Extra.equals(x.getCategory())).map(x-> (ExtraFieldEntity)x).toList();
                 m.getFields().addAll(this.builderFactory.builder(ExtraFieldBuilder.class).authorize(this.authorize).build(fieldsFields, extraFieldEntities));
+                List<ReferenceTypeFieldEntity> referenceFieldEntities = d.getFields().stream().filter(x-> DmpBlueprintFieldCategory.ReferenceType.equals(x.getCategory())).map(x-> (ReferenceTypeFieldEntity)x).toList();
+                m.getFields().addAll(this.builderFactory.builder(ReferenceFieldBuilder.class).authorize(this.authorize).build(fieldsFields, referenceFieldEntities));
             }
             models.add(m);
         }

dmp-backend/core/src/main/java/eu/eudat/model/builder/filetransformer/DescriptionTemplateDefinitionFileTransformerBuilder.java

@@ -161,14 +161,14 @@ public class DescriptionTemplateDefinitionFileTransformerBuilder extends BaseFil
 
     public BaseFieldDataFileTransformerModel convertData (BaseFieldDataEntity data) {
             BaseFieldDataFileTransformerModel m;
-            switch (data) {
+            switch (data) { //TODO new reference logic
-                case ExternalSelectDataEntity d -> {
+//                case ExternalSelectDataEntity d -> {
-                    m = new ExternalSelectDataFileTransformerModel();
+//                    m = new ExternalSelectDataFileTransformerModel();
-                    m.setLabel(d.getLabel());
+//                    m.setLabel(d.getLabel());
-                    m.setFieldType(FieldType.of(d.getFieldType().getValue()));
+//                    m.setFieldType(FieldType.of(d.getFieldType().getValue()));
-                    ((ExternalSelectDataFileTransformerModel)m).setMultipleSelect(d.getMultipleSelect());
+//                    ((ExternalSelectDataFileTransformerModel)m).setMultipleSelect(d.getMultipleSelect());
-                    ((ExternalSelectDataFileTransformerModel) m).setSources(convertAutoCompleteSingleData(d.getSources()));
+//                    ((ExternalSelectDataFileTransformerModel) m).setSources(convertAutoCompleteSingleData(d.getSources()));
-                }
+//                }
                 case ExternalDatasetDataEntity d -> {
                     m = new ExternalDatasetDataFileTransformerModel();
                     m.setLabel(d.getLabel());
@@ -221,42 +221,6 @@ public class DescriptionTemplateDefinitionFileTransformerBuilder extends BaseFil
         return result;
     }
 
-    private List<ExternalSelectDataFileTransformerModel.ExternalSelectSourceFileTransformerModel> convertAutoCompleteSingleData(List<ExternalSelectDataEntity.ExternalSelectSourceEntity> data) {
-        List<ExternalSelectDataFileTransformerModel.ExternalSelectSourceFileTransformerModel> result = new ArrayList<>();
-        for (ExternalSelectDataEntity.ExternalSelectSourceEntity d : data) {
-            ExternalSelectDataFileTransformerModel.ExternalSelectSourceFileTransformerModel m = new ExternalSelectDataFileTransformerModel.ExternalSelectSourceFileTransformerModel();
-            m.setUrl(d.getUrl());
-            m.setMethod(d.getMethod());
-            m.setHasAuth(d.getHasAuth());
-            m.setOptionsRoot(d.getOptionsRoot());
-            if (d.getAuth() != null) {
-                m.setAuth(convertAuthData(d.getAuth()));
-            }
-            m.setSourceBinding(convertComboBoxOption(d.getSourceBinding()));
-            result.add(m);
-        }
-        return result;
-    }
-
-    private ExternalSelectDataFileTransformerModel.ExternalSelectSourceBindingFileTransformerModel convertComboBoxOption(ExternalSelectDataEntity.ExternalSelectSourceBindingEntity data) {
-        ExternalSelectDataFileTransformerModel.ExternalSelectSourceBindingFileTransformerModel m = new ExternalSelectDataFileTransformerModel.ExternalSelectSourceBindingFileTransformerModel();
-//        m.setUri(data.getUri()); 
-        m.setSource(data.getSource());
-        m.setLabel(data.getLabel());
-        m.setValue(data.getValue());
-        return m;
-    }
-
-    private ExternalSelectDataFileTransformerModel.ExternalSelectAuthDataFileTransformerModel convertAuthData(ExternalSelectDataEntity.ExternalSelectAuthDataEntity data) {
-        ExternalSelectDataFileTransformerModel.ExternalSelectAuthDataFileTransformerModel m = new ExternalSelectDataFileTransformerModel.ExternalSelectAuthDataFileTransformerModel();
-        m.setMethod(data.getMethod());
-        m.setUrl(data.getUrl());
-        m.setType(data.getType());
-        m.setBody(data.getBody());
-        m.setPath(data.getPath());
-        return m;
-    }
-
     private SelectDataFileTransformerModel.OptionFileTransformerModel convertComboBoxOption(SelectDataEntity.OptionEntity data) {
         SelectDataFileTransformerModel.OptionFileTransformerModel m = new SelectDataFileTransformerModel.OptionFileTransformerModel();
         m.setLabel(data.getLabel());

dmp-backend/core/src/main/java/eu/eudat/model/builder/filetransformer/DmpBlueprintFileTransformerBuilder.java

@@ -6,21 +6,17 @@ import eu.eudat.commons.JsonHandlingService;
 import eu.eudat.commons.XmlHandlingService;
 import eu.eudat.commons.types.dmpblueprint.*;
 import eu.eudat.convention.ConventionService;
-import eu.eudat.data.DescriptionTemplateEntity;
 import eu.eudat.data.DmpBlueprintEntity;
 import eu.eudat.file.transformer.enums.DmpBlueprintExtraFieldDataType;
-import eu.eudat.file.transformer.enums.DmpBlueprintFieldCategory;
 import eu.eudat.file.transformer.enums.DmpBlueprintSystemFieldType;
-import eu.eudat.file.transformer.models.descriptiontemplate.DescriptionTemplateFileTransformerModel;
 import eu.eudat.file.transformer.models.dmpblueprint.DmpBlueprintFileTransformerModel;
 import eu.eudat.file.transformer.models.dmpblueprint.definition.*;
-import eu.eudat.model.dmpblueprintdefinition.*;
-import eu.eudat.query.DescriptionTemplateQuery;
 import eu.eudat.query.DmpBlueprintQuery;
 import gr.cite.tools.data.builder.BuilderFactory;
 import gr.cite.tools.data.query.QueryFactory;
 import gr.cite.tools.exception.MyApplicationException;
 import gr.cite.tools.logging.LoggerService;
+import org.apache.commons.lang3.NotImplementedException;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.config.ConfigurableBeanFactory;
@@ -110,6 +106,8 @@ public class DmpBlueprintFileTransformerBuilder extends BaseFileTransformerBuild
             FieldFileTransformerModel m = switch (fieldEntity.getCategory()) {
                 case System -> new SystemFieldFileTransformerModel();
                 case Extra -> new ExtraFieldFileTransformerModelFileTransformerModel();
+                case ReferenceType -> throw new NotImplementedException("ReferenceFieldFileTransformerModel");
+                default -> throw new MyApplicationException("unrecognized type " + fieldEntity.getCategory());
             };
             m.setId(fieldEntity.getId());
             m.setLabel(fieldEntity.getLabel());
@@ -120,7 +118,8 @@ public class DmpBlueprintFileTransformerBuilder extends BaseFileTransformerBuild
             switch (m) {
                 case SystemFieldFileTransformerModel sm -> sm.setSystemFieldType(DmpBlueprintSystemFieldType.of(((SystemFieldEntity)fieldEntity).getType().getValue()));
                 case ExtraFieldFileTransformerModelFileTransformerModel sm -> sm.setDataType(DmpBlueprintExtraFieldDataType.of(((ExtraFieldEntity)fieldEntity).getType().getValue()));
-                default -> {}
+//                case ReferenceFieldFileTransformerModelFileTransformerModel sm -> sm.setReferenceTypeId(((ReferenceFieldEntity)fieldEntity).getReferenceTypeId());
+                default -> throw new MyApplicationException("unrecognized type " + fieldEntity.getCategory());
             }
             result.add(m);
         });

dmp-backend/core/src/main/java/eu/eudat/model/builder/filetransformer/ReferenceFileTransformerBuilder.java

@@ -69,23 +69,9 @@ public class ReferenceFileTransformerBuilder extends BaseFileTransformerBuilder<
                 case External -> m.setSourceType(ReferenceSourceType.External);
                 default -> throw new MyApplicationException("unrecognized type " + d.getSourceType().getValue());
             }
-            switch (d.getType()){
+
-                case Taxonomies -> m.setType(ReferenceType.Taxonomies);
+            //m.setType(d.getTypeId()); //TODO new reference logic
-                case Licenses -> m.setType(ReferenceType.Licenses);
+            
-                case Publications -> m.setType(ReferenceType.Publications);
-                case Journals -> m.setType(ReferenceType.Journals);
-                case PubRepositories -> m.setType(ReferenceType.PubRepositories);
-                case DataRepositories -> m.setType(ReferenceType.DataRepositories);
-                case Registries -> m.setType(ReferenceType.Registries);
-                case Services -> m.setType(ReferenceType.Services);
-                case Project -> m.setType(ReferenceType.Project);
-                case Funder -> m.setType(ReferenceType.Funder);
-                case Datasets -> m.setType(ReferenceType.Datasets);
-                case Organizations -> m.setType(ReferenceType.Organizations);
-                case Grants -> m.setType(ReferenceType.Grants);
-                case Researcher -> m.setType(ReferenceType.Researcher);
-                default -> throw new MyApplicationException("unrecognized type " + d.getType().getValue());
-            }
             models.add(new FileTransformerBuilderItemResponse<>(m, d));
         }
         this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));

dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/FieldSearchBuilder.java

@@ -1,89 +0,0 @@
-package eu.eudat.model.builder.referencesearch;
-
-import eu.eudat.authorization.AuthorizationFlags;
-import eu.eudat.commons.enums.ReferenceFieldDataType;
-import eu.eudat.commons.enums.ReferenceType;
-import eu.eudat.configurations.referencetype.ReferenceTypeField;
-import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
-import eu.eudat.convention.ConventionService;
-import eu.eudat.model.builder.BaseBuilder;
-import eu.eudat.model.referencedefinition.Field;
-import gr.cite.tools.exception.MyApplicationException;
-import gr.cite.tools.fieldset.FieldSet;
-import gr.cite.tools.logging.DataLogEntry;
-import gr.cite.tools.logging.LoggerService;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.config.ConfigurableBeanFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import java.util.*;
-
-@Component
-@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
-public class FieldSearchBuilder extends BaseBuilder<Field, Map<String, String>> {
-
-    private final ReferenceTypeProperties properties;
-    private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
-
-    @Autowired
-    public FieldSearchBuilder(
-            ConventionService conventionService, ReferenceTypeProperties properties) {
-        super(conventionService, new LoggerService(LoggerFactory.getLogger(FieldSearchBuilder.class)));
-        this.properties = properties;
-    }
-
-    public FieldSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
-        this.authorize = values;
-        return this;
-    }
-
-    @Override
-    public List<Field> build(FieldSet fields, List< Map<String, String>> data) throws MyApplicationException {
-        this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
-        this.logger.trace(new DataLogEntry("requested fields", fields));
-        if (data == null)
-            return new ArrayList<>();
-
-        List<Field> models = new ArrayList<>();
-        for (Map<String, String> d : data) {
-
-            ReferenceType referenceType = ReferenceType.valueOf(d.getOrDefault("referenceType", null));
-            List<ReferenceTypeField> typeFields = this.getPropertiesFields(referenceType);
-            if (typeFields.isEmpty()){
-                return new ArrayList<>();
-            }
-
-            for (ReferenceTypeField typeField: typeFields){
-                Field m = new Field();
-                m.setCode(typeField.getCode());
-                m.setDataType(ReferenceFieldDataType.valueOf(typeField.getDataType()));
-                m.setValue(d.getOrDefault(typeField.getCode(), null));
-                models.add(m);
-            }
-        }
-        this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
-        return models;
-    }
-
-    private List<ReferenceTypeField> getPropertiesFields(ReferenceType referenceType){
-        switch (referenceType) {
-            case Taxonomies: return properties.getTaxonomy().get("fields");
-            case Licenses: return properties.getLicence().get("fields");
-            case Publications: return properties.getPublication().get("fields");
-            case Journals: return properties.getJournal().get("fields");
-            case PubRepositories: return properties.getPubRepository().get("fields");
-            case DataRepositories: return properties.getDataRepository().get("fields");
-            case Registries: return properties.getRegistry().get("fields");
-            case Services: return properties.getService().get("fields");
-            case Organizations: return properties.getOrganisation().get("fields");
-            case Datasets: return properties.getDataset().get("fields");
-            case Funder: return properties.getFunder().get("fields");
-            case Grants: return properties.getGrant().get("fields");
-            case Project: return properties.getProject().get("fields");
-            case Researcher: return properties.getResearcher().get("fields");
-            default: return null;
-        }
-    }
-}

dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/ReferenceSearchBuilder.java

@@ -1,103 +0,0 @@
-package eu.eudat.model.builder.referencesearch;
-
-import eu.eudat.authorization.AuthorizationFlags;
-import eu.eudat.commons.XmlHandlingService;
-import eu.eudat.commons.enums.ReferenceSourceType;
-import eu.eudat.commons.enums.ReferenceType;
-import eu.eudat.convention.ConventionService;
-import eu.eudat.model.Reference;
-import eu.eudat.model.builder.BaseBuilder;
-import eu.eudat.model.referencedefinition.Definition;
-import eu.eudat.model.referencedefinition.Field;
-import gr.cite.tools.data.builder.BuilderFactory;
-import gr.cite.tools.data.query.QueryFactory;
-import gr.cite.tools.exception.MyApplicationException;
-import gr.cite.tools.fieldset.FieldSet;
-import gr.cite.tools.logging.DataLogEntry;
-import gr.cite.tools.logging.LoggerService;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.config.ConfigurableBeanFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import java.util.*;
-
-@Component
-@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
-public class ReferenceSearchBuilder extends BaseBuilder<Reference, Map<String, String>> {
-
-    private final BuilderFactory builderFactory;
-    private final QueryFactory queryFactory;
-    private final XmlHandlingService xmlHandlingService;
-    private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
-
-    @Autowired
-    public ReferenceSearchBuilder(
-            ConventionService conventionService,
-            BuilderFactory builderFactory, QueryFactory queryFactory, XmlHandlingService xmlHandlingService) {
-        super(conventionService, new LoggerService(LoggerFactory.getLogger(ReferenceSearchBuilder.class)));
-        this.builderFactory = builderFactory;
-        this.queryFactory = queryFactory;
-        this.xmlHandlingService = xmlHandlingService;
-    }
-
-    public ReferenceSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
-        this.authorize = values;
-        return this;
-    }
-
-    @Override
-    public List<Reference> build(FieldSet fields, List<Map<String, String>> data) throws MyApplicationException {
-        this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
-        this.logger.trace(new DataLogEntry("requested fields", fields));
-        if (data == null)
-            return new ArrayList<>();
-
-        List<Reference> models = new ArrayList<>();
-        for (Map<String, String> d : data) {
-            Reference m = new Reference();
-            if (d.containsKey("id")) m.setId(UUID.fromString(d.getOrDefault("id", null)));
-            m.setLabel(d.getOrDefault("name", null));
-            m.setSourceType(ReferenceSourceType.External);
-            String type = d.getOrDefault("referenceType", null);
-            if (type != null) m.setType(ReferenceType.valueOf(type));
-
-            Definition definition = this.builderFactory.builder(DefinitionSearchBuilder.class).authorize(this.authorize).build(null, d);
-            m.setDefinition(definition);
-
-            switch (m.getType()){
-                case Researcher:
-                case Organizations:
-                case Funder:
-                case Grants:
-                case DataRepositories:
-                case PubRepositories:
-                case Journals:
-                case Datasets:
-                case Registries:
-                case Services:{
-                    for (Field field : definition.getFields()) {
-                        if (field.getCode().equals("pid") && field.getValue() != null) {
-                            if (d.containsKey("key")) {
-                                m.setReference(d.get("key") + ":" + field.getValue());
-                            }
-                        }
-                        if (field.getCode().equals("tag") && field.getValue() != null){
-                            m.setSource(field.getValue());
-                        }
-                    }
-                }
-            }
-
-            m.setDescription(d.getOrDefault("description", null));
-            m.setAbbreviation(d.getOrDefault("abbreviation", null));
-
-            models.add(m);
-
-        }
-        this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
-        return models;
-    }
-
-}

dmp-backend/core/src/main/java/eu/eudat/model/censorship/PublicReferenceCensor.java

@@ -2,6 +2,8 @@ package eu.eudat.model.censorship;
 
 import eu.eudat.authorization.Permission;
 import eu.eudat.convention.ConventionService;
+import eu.eudat.model.PublicDmpUser;
+import eu.eudat.model.PublicReference;
 import eu.eudat.model.Reference;
 import eu.eudat.model.censorship.referencedefinition.DefinitionCensor;
 import gr.cite.commons.web.authz.service.AuthorizationService;
@@ -39,6 +41,8 @@ public class PublicReferenceCensor extends BaseCensor {
             return;
 
         this.authService.authorizeForce(Permission.PublicBrowseReference);
+        FieldSet typeFields = fields.extractPrefixed(this.asIndexerPrefix(PublicReference._type));
+        this.censorFactory.censor(PublicReferenceCensor.class).censor(typeFields);
     }
 
 }

dmp-backend/core/src/main/java/eu/eudat/model/censorship/PublicReferenceTypeCensor.java

@@ -0,0 +1,41 @@
+package eu.eudat.model.censorship;
+
+import eu.eudat.authorization.Permission;
+import eu.eudat.convention.ConventionService;
+import eu.eudat.model.PublicDmpUser;
+import gr.cite.commons.web.authz.service.AuthorizationService;
+import gr.cite.tools.data.censor.CensorFactory;
+import gr.cite.tools.fieldset.FieldSet;
+import gr.cite.tools.logging.DataLogEntry;
+import gr.cite.tools.logging.LoggerService;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+@Component
+@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+public class PublicReferenceTypeCensor extends BaseCensor {
+
+    private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(PublicReferenceTypeCensor.class));
+
+    protected final AuthorizationService authService;
+    protected final CensorFactory censorFactory;
+
+    public PublicReferenceTypeCensor(ConventionService conventionService,
+                                     AuthorizationService authService,
+                                     CensorFactory censorFactory) {
+        super(conventionService);
+        this.authService = authService;
+        this.censorFactory = censorFactory;
+    }
+
+    public void censor(FieldSet fields) {
+        logger.debug(new DataLogEntry("censoring fields", fields));
+        if (fields == null || fields.isEmpty())
+            return;
+
+        this.authService.authorizeForce(Permission.PublicBrowseReferenceType);
+    }
+
+}

dmp-backend/core/src/main/java/eu/eudat/model/censorship/ReferenceCensor.java

@@ -45,6 +45,8 @@ public class ReferenceCensor extends BaseCensor {
         this.censorFactory.censor(DmpReferenceCensor.class).censor(dmpReferencesFields, userId);
         FieldSet createdByFields = fields.extractPrefixed(this.asIndexerPrefix(Reference._createdBy));
         this.censorFactory.censor(UserCensor.class).censor(createdByFields, userId);
+        FieldSet typeFields = fields.extractPrefixed(this.asIndexerPrefix(Reference._type));
+        this.censorFactory.censor(ReferenceTypeCensor.class).censor(typeFields, userId);
     }
 
 }

dmp-backend/core/src/main/java/eu/eudat/model/descriptiontemplatedefinition/fielddata/ExternalSelectData.java

@@ -1,164 +0,0 @@
-package eu.eudat.model.descriptiontemplatedefinition.fielddata;
-
-import java.util.List;
-
-public class ExternalSelectData extends LabelAndMultiplicityData {
-	private List<ExternalSelectSource> sources;
-
-	public final static String _sources = "sources";
-
-	public List<ExternalSelectSource> getSources() {
-		return sources;
-	}
-
-	public void setSources(List<ExternalSelectSource> sources) {
-		this.sources = sources;
-	}
-
-	public static class ExternalSelectSource {
-		private String url;
-		public final static String _url = "url";
-		
-		private String method;
-		public final static String _method = "method";
-		private String optionsRoot;
-		public final static String _optionsRoot = "optionsRoot";
-		private ExternalSelectSourceBinding sourceBinding;
-		public final static String _sourceBinding = "sourceBinding";
-		private Boolean hasAuth;
-		public final static String _hasAuth = "hasAuth";
-		private ExternalSelectAuthData auth;
-		public final static String _auth = "auth";
-
-		public String getUrl() {
-			return url;
-		}
-
-		public void setUrl(String url) {
-			this.url = url;
-		}
-
-		public String getMethod() {
-			return method;
-		}
-
-		public void setMethod(String method) {
-			this.method = method;
-		}
-
-		public String getOptionsRoot() {
-			return optionsRoot;
-		}
-
-		public void setOptionsRoot(String optionsRoot) {
-			this.optionsRoot = optionsRoot;
-		}
-
-		public ExternalSelectSourceBinding getSourceBinding() {
-			return sourceBinding;
-		}
-
-		public void setSourceBinding(ExternalSelectSourceBinding sourceBinding) {
-			this.sourceBinding = sourceBinding;
-		}
-
-		public Boolean getHasAuth() {
-			return hasAuth;
-		}
-
-		public void setHasAuth(Boolean hasAuth) {
-			this.hasAuth = hasAuth;
-		}
-
-		public ExternalSelectAuthData getAuth() {
-			return auth;
-		}
-
-		public void setAuth(ExternalSelectAuthData auth) {
-			this.auth = auth;
-		}
-	}
-
-	public static class ExternalSelectSourceBinding {
-		private String label;
-		public final static String _label = "label";
-		private String value;
-		public final static String _value = "value";
-		private String source;
-		public final static String _source = "source";
-
-		public String getLabel() {
-			return label;
-		}
-		public void setLabel(String label) {
-			this.label = label;
-		}
-
-		public String getValue() {
-			return value;
-		}
-		public void setValue(String value) {
-			this.value = value;
-		}
-
-		public String getSource() {
-			return source;
-		}
-		public void setSource(String source) {
-			this.source = source;
-		}
-	}
-
-	public static class ExternalSelectAuthData {
-		private String url;
-		public final static String _url = "url";
-		private String method;
-		public final static String _method = "method";
-		private String body;
-		public final static String _body = "body";
-		private String path;
-		public final static String _path = "path";
-		private String type;
-		public final static String _type = "type";
-
-		public String getUrl() {
-			return url;
-		}
-
-		public void setUrl(String url) {
-			this.url = url;
-		}
-
-		public String getMethod() {
-			return method;
-		}
-
-		public void setMethod(String method) {
-			this.method = method;
-		}
-
-		public String getBody() {
-			return body;
-		}
-
-		public void setBody(String body) {
-			this.body = body;
-		}
-
-		public String getPath() {
-			return path;
-		}
-
-		public void setPath(String path) {
-			this.path = path;
-		}
-
-		public String getType() {
-			return type;
-		}
-
-		public void setType(String type) {
-			this.type = type;
-		}
-	}
-}

dmp-backend/core/src/main/java/eu/eudat/model/descriptiontemplatedefinition/fielddata/ReferenceTypeData.java

@@ -0,0 +1,25 @@
+package eu.eudat.model.descriptiontemplatedefinition.fielddata;
+
+import java.util.UUID;
+
+public class ReferenceTypeData extends BaseFieldData {
+	private Boolean multipleSelect;
+	public final static String _multipleSelect = "multipleSelect";
+
+	private UUID referenceTypeId;
+	public final static String _referenceTypeId = "referenceTypeId";
+	public Boolean getMultipleSelect() {
+		return multipleSelect;
+	}
+	public void setMultipleSelect(Boolean multipleSelect) {
+		this.multipleSelect = multipleSelect;
+	}
+
+	public UUID getReferenceTypeId() {
+		return referenceTypeId;
+	}
+
+	public void setReferenceTypeId(UUID referenceTypeId) {
+		this.referenceTypeId = referenceTypeId;
+	}
+}

dmp-backend/core/src/main/java/eu/eudat/model/dmpblueprintdefinition/ExtraField.java

@@ -2,6 +2,7 @@ package eu.eudat.model.dmpblueprintdefinition;
 
 import eu.eudat.commons.enums.DmpBlueprintExtraFieldDataType;
 
+
 public class ExtraField extends Field {
 
 	public final static String _dataType = "dataType";
@@ -15,3 +16,4 @@ public class ExtraField extends Field {
 		this.dataType = dataType;
 	}
 }
+

dmp-backend/core/src/main/java/eu/eudat/model/dmpblueprintdefinition/ReferenceTypeField.java

@@ -0,0 +1,19 @@
+package eu.eudat.model.dmpblueprintdefinition;
+
+import eu.eudat.model.ReferenceType;
+
+import java.util.UUID;
+
+public class ReferenceTypeField extends Field {
+
+	public final static String _referenceType = "referenceTypeId";
+	private ReferenceType referenceType;
+
+	public ReferenceType getReferenceType() {
+		return referenceType;
+	}
+
+	public void setReferenceType(ReferenceType referenceType) {
+		this.referenceType = referenceType;
+	}
+}

dmp-backend/core/src/main/java/eu/eudat/model/mapper/publicapi/DescriptionToPublicApiDatasetListingMapper.java

@@ -5,6 +5,7 @@ import eu.eudat.model.publicapi.datasetprofile.DatasetProfilePublicModel;
 import eu.eudat.model.publicapi.grant.GrantPublicOverviewModel;
 import eu.eudat.model.publicapi.listingmodels.DatasetPublicListingModel;
 import eu.eudat.model.publicapi.user.UserInfoPublicModel;
+import eu.eudat.service.publicapi.PublicApiProperties;
 import org.springframework.stereotype.Component;
 
 import java.sql.Date;
@@ -13,7 +14,7 @@ import java.util.List;
 @Component
 public class DescriptionToPublicApiDatasetListingMapper {
 
-    public DatasetPublicListingModel toPublicListingModel(Description description) {
+    public DatasetPublicListingModel toPublicListingModel(Description description, PublicApiProperties.ReferenceTypeMapConfig config) {
         DatasetPublicListingModel model = new DatasetPublicListingModel();
         model.setId(description.getId().toString());
         model.setLabel(description.getLabel());
@@ -24,7 +25,7 @@ public class DescriptionToPublicApiDatasetListingMapper {
         model.setDmpId(description.getDmp().getId().toString());
         model.setUsers(List.of(UserInfoPublicModel.fromDescriptionCreator(description.getCreatedBy())));
         model.setProfile(DatasetProfilePublicModel.fromDataModel(description.getDescriptionTemplate()));
-        model.setGrant(GrantPublicOverviewModel.fromDescriptionReference(description.getDescriptionReferences()));
+        model.setGrant(GrantPublicOverviewModel.fromDescriptionReference(description.getDescriptionReferences(), config));
 
         model.setCreatedAt(Date.from(description.getCreatedAt()));
         model.setModifiedAt(Date.from(description.getUpdatedAt()));

dmp-backend/core/src/main/java/eu/eudat/model/mapper/publicapi/DescriptionToPublicApiDatasetMapper.java

@@ -8,6 +8,7 @@ import eu.eudat.model.publicapi.datasetwizard.RegistryPublicModel;
 import eu.eudat.model.publicapi.datasetwizard.ServicePublicModel;
 import eu.eudat.model.publicapi.listingmodels.DataManagementPlanPublicListingModel;
 import eu.eudat.model.publicapi.overviewmodels.DatasetPublicModel;
+import eu.eudat.service.publicapi.PublicApiProperties;
 import org.springframework.stereotype.Component;
 
 import java.sql.Date;
@@ -22,7 +23,7 @@ public class DescriptionToPublicApiDatasetMapper {
         this.descriptionTemplateToPublicApiDatasetProfileMapper = descriptionTemplateToPublicApiDatasetProfileMapper;
     }
 
-    public DatasetPublicModel toPublicModel(Description description, DataManagementPlanPublicListingModel dmp) {
+    public DatasetPublicModel toPublicModel(Description description, DataManagementPlanPublicListingModel dmp, PublicApiProperties.ReferenceTypeMapConfig config) {
         DatasetPublicModel model = new DatasetPublicModel();
         model.setId(description.getId());
         model.setLabel(description.getLabel());
@@ -36,10 +37,10 @@ public class DescriptionToPublicApiDatasetMapper {
 
         model.setProfile(DatasetProfilePublicModel.fromDataModel(description.getDescriptionTemplate()));
         if (description.getDescriptionReferences() != null) {
-            model.setRegistries(description.getDescriptionReferences().stream().map(RegistryPublicModel::fromDescriptionReference).filter(Objects::nonNull).toList());
+            model.setRegistries(description.getDescriptionReferences().stream().map(x-> RegistryPublicModel.fromDescriptionReference(x, config)).filter(Objects::nonNull).toList());
-            model.setServices(description.getDescriptionReferences().stream().map(ServicePublicModel::fromDescriptionReference).filter(Objects::nonNull).toList());
+            model.setServices(description.getDescriptionReferences().stream().map(x-> ServicePublicModel.fromDescriptionReference(x, config)).filter(Objects::nonNull).toList());
-            model.setDataRepositories(description.getDescriptionReferences().stream().map(DataRepositoryPublicModel::fromDescriptionReference).filter(Objects::nonNull).toList());
+            model.setDataRepositories(description.getDescriptionReferences().stream().map(x-> DataRepositoryPublicModel.fromDescriptionReference(x, config)).filter(Objects::nonNull).toList());
-            model.setExternalDatasets(description.getDescriptionReferences().stream().map(ExternalDatasetPublicListingModel::fromDescriptionReference).filter(Objects::nonNull).toList());
+            model.setExternalDatasets(description.getDescriptionReferences().stream().map(x-> ExternalDatasetPublicListingModel.fromDescriptionReference(x, config)).filter(Objects::nonNull).toList());
         }
 
         model.setCreatedAt(Date.from(description.getCreatedAt()));

dmp-backend/core/src/main/java/eu/eudat/model/mapper/publicapi/DmpToPublicApiDmpListingMapper.java

@@ -4,6 +4,7 @@ import eu.eudat.model.Dmp;
 import eu.eudat.model.publicapi.listingmodels.DataManagementPlanPublicListingModel;
 import eu.eudat.model.publicapi.researcher.ResearcherPublicModel;
 import eu.eudat.model.publicapi.user.UserInfoPublicModel;
+import eu.eudat.service.publicapi.PublicApiProperties;
 import org.springframework.stereotype.Component;
 
 import java.sql.Date;
@@ -12,7 +13,7 @@ import java.util.Objects;
 @Component
 public class DmpToPublicApiDmpListingMapper {
 
-    public DataManagementPlanPublicListingModel toPublicListingModel(Dmp dmp) {
+    public DataManagementPlanPublicListingModel toPublicListingModel(Dmp dmp, PublicApiProperties.ReferenceTypeMapConfig config) {
         DataManagementPlanPublicListingModel model = new DataManagementPlanPublicListingModel();
         model.setId(dmp.getId().toString());
         model.setLabel(dmp.getLabel());
@@ -20,7 +21,7 @@ public class DmpToPublicApiDmpListingMapper {
         model.setGroupId(dmp.getGroupId());
 
         model.setUsers(dmp.getDmpUsers().stream().map(UserInfoPublicModel::fromDmpUser).toList());
-        model.setResearchers(dmp.getDmpReferences().stream().map(ResearcherPublicModel::fromDmpReference).filter(Objects::nonNull).toList());
+        model.setResearchers(dmp.getDmpReferences().stream().map(x-> ResearcherPublicModel.fromDmpReference(x, config)).filter(Objects::nonNull).toList());
 
         model.setCreatedAt(Date.from(dmp.getCreatedAt()));
         model.setModifiedAt(Date.from(dmp.getUpdatedAt()));

dmp-backend/core/src/main/java/eu/eudat/model/mapper/publicapi/DmpToPublicApiDmpMapper.java

@@ -11,6 +11,7 @@ import eu.eudat.model.publicapi.organisation.OrganizationPublicModel;
 import eu.eudat.model.publicapi.overviewmodels.DataManagementPlanPublicModel;
 import eu.eudat.model.publicapi.researcher.ResearcherPublicModel;
 import eu.eudat.model.publicapi.user.UserInfoPublicModel;
+import eu.eudat.service.publicapi.PublicApiProperties;
 import org.springframework.stereotype.Component;
 
 import java.sql.Date;
@@ -26,7 +27,7 @@ public class DmpToPublicApiDmpMapper {
         this.descriptionToPublicApiDatasetMapper = descriptionToPublicApiDatasetMapper;
     }
 
-    public DataManagementPlanPublicModel toPublicModel(Dmp dmp, List<EntityDoiEntity> doiEntities, List<DmpDescriptionTemplate> descriptionTemplates) {
+    public DataManagementPlanPublicModel toPublicModel(Dmp dmp, List<EntityDoiEntity> doiEntities, List<DmpDescriptionTemplate> descriptionTemplates, PublicApiProperties.ReferenceTypeMapConfig config) {
         DataManagementPlanPublicModel model = new DataManagementPlanPublicModel();
         model.setId(dmp.getId().toString());
         model.setLabel(dmp.getLabel());
@@ -38,13 +39,13 @@ public class DmpToPublicApiDmpMapper {
         if (dmp.getDescriptions() != null) {
             DataManagementPlanPublicListingModel publicListingModel = new DataManagementPlanPublicListingModel();
             publicListingModel.setId(model.getId());
-            model.setDatasets(dmp.getDescriptions().stream().map(x -> descriptionToPublicApiDatasetMapper.toPublicModel(x, publicListingModel)).toList());
+            model.setDatasets(dmp.getDescriptions().stream().map(x -> descriptionToPublicApiDatasetMapper.toPublicModel(x, publicListingModel, config)).toList());
         }
 
         model.setUsers(dmp.getDmpUsers().stream().map(UserInfoPublicModel::fromDmpUser).toList());
-        model.setResearchers(dmp.getDmpReferences().stream().map(ResearcherPublicModel::fromDmpReference).filter(Objects::nonNull).toList());
+        model.setResearchers(dmp.getDmpReferences().stream().map(x-> ResearcherPublicModel.fromDmpReference(x, config)).filter(Objects::nonNull).toList());
-        model.setGrant(GrantPublicOverviewModel.fromDmpReferences(dmp.getDmpReferences()));
+        model.setGrant(GrantPublicOverviewModel.fromDmpReferences(dmp.getDmpReferences(), config));
-        model.setOrganisations(dmp.getDmpReferences().stream().map(OrganizationPublicModel::fromDmpReference).filter(Objects::nonNull).toList());
+        model.setOrganisations(dmp.getDmpReferences().stream().map(x-> OrganizationPublicModel.fromDmpReference(x, config)).filter(Objects::nonNull).toList());
         model.setDois(doiEntities.stream().map(DoiPublicModel::fromDataModel).toList());
         model.setAssociatedProfiles(descriptionTemplates.stream().map(x -> AssociatedProfilePublicModel.fromDmpDescriptionTemplate(x, dmp.getBlueprint())).toList());
 

dmp-backend/core/src/main/java/eu/eudat/model/persist/ReferencePersist.java

@@ -1,7 +1,6 @@
 package eu.eudat.model.persist;
 
 import eu.eudat.commons.enums.ReferenceSourceType;
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.commons.validation.BaseValidator;
 import gr.cite.tools.validation.ValidatorFactory;
 import gr.cite.tools.validation.specification.Specification;
@@ -27,9 +26,9 @@ public class ReferencePersist {
 
     public static final String _label = "label";
 
-    private ReferenceType type;
+    private UUID typeId;
 
-    public static final String _type = "type";
+    public static final String _typeId = "typeId";
 
     private String description;
 
@@ -73,12 +72,12 @@ public class ReferencePersist {
         this.label = label;
     }
 
-    public ReferenceType getType() {
+    public UUID getTypeId() {
-        return type;
+        return typeId;
     }
 
-    public void setType(ReferenceType type) {
+    public void setTypeId(UUID typeId) {
-        this.type = type;
+        this.typeId = typeId;
     }
 
     public String getDescription() {
@@ -177,8 +176,8 @@ public class ReferencePersist {
                             .must(() -> this.lessEqualLength(item.getLabel(), ReferenceEntity._labelLength))
                             .failOn(ReferencePersist._label).failWith(messageSource.getMessage("Validation_MaxLength", new Object[]{ReferencePersist._label}, LocaleContextHolder.getLocale())),
                     this.spec()
-                            .must(() -> !this.isNull(item.getType()))
+                            .must(() -> this.isValidGuid(item.getTypeId()))
-                            .failOn(ReferencePersist._type).failWith(messageSource.getMessage("Validation_Required", new Object[]{ReferencePersist._type}, LocaleContextHolder.getLocale())),
+                            .failOn(ReferencePersist._typeId).failWith(messageSource.getMessage("Validation_Required", new Object[]{ReferencePersist._typeId}, LocaleContextHolder.getLocale())),
                     this.spec()
                             .must(() -> !this.isEmpty(item.getReference()))
                             .failOn(ReferencePersist._reference).failWith(messageSource.getMessage("Validation_Required", new Object[]{ReferencePersist._reference}, LocaleContextHolder.getLocale())),
@@ -200,7 +199,6 @@ public class ReferencePersist {
                     this.spec()
                             .must(() -> !this.isNull(item.getSourceType()))
                             .failOn(ReferencePersist._sourceType).failWith(messageSource.getMessage("Validation_Required", new Object[]{ReferencePersist._sourceType}, LocaleContextHolder.getLocale())),
-
                    this.refSpec()
                             .iff(() -> !this.isNull(item.getDefinition()))
                             .on(ReferencePersist._definition)

dmp-backend/core/src/main/java/eu/eudat/model/persist/descriptionproperties/FieldPersist.java

@@ -97,10 +97,6 @@ public class FieldPersist {
         protected List<Specification> specifications(FieldPersist item) {
             FieldType fieldType = this.fieldEntity != null && this.fieldEntity.getData() != null ? this.fieldEntity.getData().getFieldType() :  FieldType.FREE_TEXT;
 	        return Arrays.asList(
-                    this.spec()
-                            .iff(()-> FieldType.isNotImplemented(fieldType))
-                            .must(() -> false)
-                            .failOn(FieldPersist._textValue).failWith("Field Not implemented yet!!"),
                     this.spec()
                             .iff(()-> FieldType.isTextType(fieldType))
                             .must(() -> !this.isEmpty(item.getTextValue()))
@@ -130,7 +126,7 @@ public class FieldPersist {
                             .must(() -> this.isUUID(item.getTextValue()))
                             .failOn(FieldPersist._textValue).failWith(messageSource.getMessage("Validation_UnexpectedValue", new Object[]{FieldPersist._textValue}, LocaleContextHolder.getLocale())),
                     this.spec()
-                            .iff(()-> !this.isNull(item.getTextListValue()) && (fieldType.equals(FieldType.INTERNAL_DMP_ENTRIES_DMPS) || fieldType.equals(FieldType.INTERNAL_DMP_ENTRIES_RESEARCHERS) || fieldType.equals(FieldType.INTERNAL_DMP_ENTRIES_DATASETS)))
+                            .iff(()-> !this.isNull(item.getTextListValue()) && (fieldType.equals(FieldType.INTERNAL_DMP_ENTRIES_DMPS)  || fieldType.equals(FieldType.INTERNAL_DMP_ENTRIES_DATASETS)))
                             .must(() -> item.getTextListValue().stream().allMatch(this::isUUID))
                             .failOn(FieldPersist._textListValue).failWith(messageSource.getMessage("Validation_UnexpectedValue", new Object[]{FieldPersist._textListValue}, LocaleContextHolder.getLocale())),
                     this.navSpec()

dmp-backend/core/src/main/java/eu/eudat/model/persist/descriptiontemplatedefinition/fielddata/BaseFieldDataPersist.java

@@ -21,7 +21,6 @@ import java.util.List;
         visible = true,
         defaultImpl = LabelDataPersist.class)
 @JsonSubTypes({
-        @JsonSubTypes.Type(value = ExternalSelectDataPersist.class, name = FieldType.Names.ExternalSelect),
         @JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.BooleanDecision),
         @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.InternalDmpDescriptions),
         @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.InternalDmpDmps),
@@ -29,26 +28,16 @@ import java.util.List;
         @JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.DatePicker),
         @JsonSubTypes.Type(value = ExternalDatasetDataPersist.class, name = FieldType.Names.ExternalDatasets),
         @JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.FreeText),
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.Licenses),
+        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.ReferenceTypes),
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.Organizations),
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.Publications),
         @JsonSubTypes.Type(value = RadioBoxDataPersist.class, name = FieldType.Names.RadioBox),
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.Registries),
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.InternalDmpResearchers),
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.Researchers),
         @JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.RichTextarea),
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.Services),
         @JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.Tags),
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.Taxonomies),
         @JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.TextArea),
         @JsonSubTypes.Type(value = UploadDataPersist.class, name = FieldType.Names.Upload),
         @JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.Validation),
         @JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.DatasetIdentifier),
         @JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.Currency),
-        @JsonSubTypes.Type(value = SelectDataPersist.class, name = FieldType.Names.Select),
+        @JsonSubTypes.Type(value = SelectDataPersist.class, name = FieldType.Names.Select)
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.DataRepositories),
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.JournalRepositories),
-        @JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.PubRepositories),
 })
 public abstract class BaseFieldDataPersist {
 

dmp-backend/core/src/main/java/eu/eudat/model/persist/descriptiontemplatedefinition/fielddata/ReferenceTypeDataPersist.java

@@ -0,0 +1,71 @@
+package eu.eudat.model.persist.descriptiontemplatedefinition.fielddata;
+
+import eu.eudat.convention.ConventionService;
+import eu.eudat.errorcode.ErrorThesaurusProperties;
+import gr.cite.tools.validation.specification.Specification;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.MessageSource;
+import org.springframework.context.annotation.Scope;
+import org.springframework.context.i18n.LocaleContextHolder;
+import org.springframework.stereotype.Component;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
+public class ReferenceTypeDataPersist extends BaseFieldDataPersist {
+
+    private Boolean multipleSelect;
+
+    public static final String _multipleSelect = "multipleSelect";
+
+    private UUID referenceTypeId;
+    public final static String _referenceTypeId = "referenceTypeId";
+
+    public Boolean getMultipleSelect() {
+        return multipleSelect;
+    }
+
+    public void setMultipleSelect(Boolean multipleSelect) {
+        this.multipleSelect = multipleSelect;
+    }
+
+    public UUID getReferenceTypeId() {
+        return referenceTypeId;
+    }
+
+    public void setReferenceTypeId(UUID referenceTypeId) {
+        this.referenceTypeId = referenceTypeId;
+    }
+
+    @Component(ReferenceTypeDataPersist.ReferenceTypeDataPersistValidator.ValidatorName)
+    @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+    public static class ReferenceTypeDataPersistValidator extends BaseFieldDataPersistValidator<ReferenceTypeDataPersist> {
+
+        public static final String ValidatorName = "DescriptionTemplate.ReferenceTypeDataPersistValidator";
+
+        protected ReferenceTypeDataPersistValidator(ConventionService conventionService, ErrorThesaurusProperties errors, MessageSource messageSource) {
+            super(conventionService, errors, messageSource);
+        }
+
+        @Override
+        protected Class<ReferenceTypeDataPersist> modelClass() {
+            return ReferenceTypeDataPersist.class;
+        }
+
+        @Override
+        protected List<Specification> specifications(ReferenceTypeDataPersist item) {
+            List<Specification> specifications = getBaseSpecifications(item);
+            specifications.addAll(Arrays.asList(
+                    this.spec()
+                            .must(() -> this.isValidGuid(item.getReferenceTypeId()))
+                            .failOn(ReferenceTypeDataPersist._referenceTypeId).failWith(messageSource.getMessage("Validation_Required", new Object[]{ReferenceTypeDataPersist._referenceTypeId}, LocaleContextHolder.getLocale())),
+                    this.spec()
+                            .must(() -> !this.isNull(item.getMultipleSelect()))
+                            .failOn(ReferenceTypeDataPersist._multipleSelect).failWith(messageSource.getMessage("Validation_Required", new Object[]{ReferenceTypeDataPersist._multipleSelect}, LocaleContextHolder.getLocale()))
+            ));
+            return specifications;
+        }
+    }
+
+}

dmp-backend/core/src/main/java/eu/eudat/model/persist/dmpblueprintdefinition/ReferenceTypeFieldPersist.java

@@ -0,0 +1,56 @@
+package eu.eudat.model.persist.dmpblueprintdefinition;
+
+import eu.eudat.convention.ConventionService;
+import eu.eudat.errorcode.ErrorThesaurusProperties;
+import gr.cite.tools.validation.specification.Specification;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.MessageSource;
+import org.springframework.context.annotation.Scope;
+import org.springframework.context.i18n.LocaleContextHolder;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.UUID;
+
+public class ReferenceTypeFieldPersist extends FieldPersist {
+
+    private UUID referenceTypeId;
+
+    public static final String _referenceTypeId = "referenceTypeId";
+
+    public UUID getReferenceTypeId() {
+        return referenceTypeId;
+    }
+
+    public void setReferenceTypeId(UUID referenceTypeId) {
+        this.referenceTypeId = referenceTypeId;
+    }
+
+    @Component(ReferenceFieldPersistPersistValidator.ValidatorName)
+    @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+    public static class ReferenceFieldPersistPersistValidator extends BaseFieldPersistValidator<ReferenceTypeFieldPersist> {
+
+        public static final String ValidatorName = "DmpBlueprint.ReferenceFieldPersistPersistValidator";
+
+        protected ReferenceFieldPersistPersistValidator(ConventionService conventionService, ErrorThesaurusProperties errors, MessageSource messageSource) {
+            super(conventionService, errors, messageSource);
+        }
+
+        @Override
+        protected Class<ReferenceTypeFieldPersist> modelClass() {
+            return ReferenceTypeFieldPersist.class;
+        }
+
+        @Override
+        protected List<Specification> specifications(ReferenceTypeFieldPersist item) {
+            List<Specification> specifications = getBaseSpecifications(item);
+            specifications.add(
+                    this.spec()
+                            .must(() -> !this.isNull(item.getReferenceTypeId()))
+                            .failOn(ReferenceTypeFieldPersist._referenceTypeId).failWith(messageSource.getMessage("Validation_Required", new Object[]{ReferenceTypeFieldPersist._referenceTypeId}, LocaleContextHolder.getLocale()))
+            );
+            return specifications;
+        }
+    }
+
+}

dmp-backend/core/src/main/java/eu/eudat/model/persist/dmpblueprintdefinition/SectionPersist.java

@@ -1,7 +1,7 @@
 package eu.eudat.model.persist.dmpblueprintdefinition;
 
-import eu.eudat.commons.enums.DmpBlueprintFieldCategory;
 import eu.eudat.commons.validation.BaseValidator;
+import gr.cite.tools.exception.MyApplicationException;
 import gr.cite.tools.validation.ValidatorFactory;
 import gr.cite.tools.validation.specification.Specification;
 import eu.eudat.convention.ConventionService;
@@ -13,7 +13,6 @@ import org.springframework.context.i18n.LocaleContextHolder;
 import org.springframework.stereotype.Component;
 
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -145,7 +144,20 @@ public class SectionPersist {
                             .iff(() -> !this.isListNullOrEmpty(item.getFields()))
                             .on(SectionPersist._fields)
                             .over(item.getFields())
-                            .using((itm) -> ((FieldPersist) itm).getCategory() == DmpBlueprintFieldCategory.Extra ? this.validatorFactory.validator(ExtraFieldPersist.ExtraFieldPersistValidator.class) : this.validatorFactory.validator(SystemFieldPersist.SystemFieldPersistValidator.class)),
+                            .using((itm) -> {
+                                switch (((FieldPersist) itm).getCategory()){
+                                    case Extra -> {
+                                        return this.validatorFactory.validator(ExtraFieldPersist.ExtraFieldPersistValidator.class);
+                                    }
+                                    case System -> {
+                                        return this.validatorFactory.validator(SystemFieldPersist.SystemFieldPersistValidator.class);
+                                    }
+                                    case ReferenceType -> {
+                                        return this.validatorFactory.validator(ReferenceTypeFieldPersist.ReferenceFieldPersistPersistValidator.class);
+                                    }
+                                    default -> throw new MyApplicationException("unrecognized type " + ((FieldPersist) itm).getCategory());
+                                }
+                            }),
                     this.navSpec()
                             .iff(() -> !this.isListNullOrEmpty(item.getDescriptionTemplates()))
                             .on(SectionPersist._descriptionTemplates)

dmp-backend/core/src/main/java/eu/eudat/model/persist/dmpblueprintdefinition/SystemFieldPersist.java

@@ -11,6 +11,7 @@ import org.springframework.context.i18n.LocaleContextHolder;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
+import java.util.UUID;
 
 public class SystemFieldPersist extends FieldPersist {
 
@@ -54,3 +55,4 @@ public class SystemFieldPersist extends FieldPersist {
     }
 
 }
+

dmp-backend/core/src/main/java/eu/eudat/model/publicapi/datasetwizard/DataRepositoryPublicModel.java

@@ -1,12 +1,12 @@
 package eu.eudat.model.publicapi.datasetwizard;
 
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.model.DescriptionReference;
 import eu.eudat.model.Reference;
 import eu.eudat.model.referencedefinition.Field;
+import eu.eudat.service.publicapi.PublicApiProperties;
+
+import java.util.Objects;
 
-import java.util.Date;
-import java.util.UUID;
 
 public class DataRepositoryPublicModel {
     private String id;
@@ -90,8 +90,8 @@ public class DataRepositoryPublicModel {
         return null;
     }
 
-    public static DataRepositoryPublicModel fromDescriptionReference(DescriptionReference descriptionReference) {
+    public static DataRepositoryPublicModel fromDescriptionReference(DescriptionReference descriptionReference, PublicApiProperties.ReferenceTypeMapConfig config) {
-        if (descriptionReference.getReference().getType() != ReferenceType.DataRepositories)
+        if (!Objects.equals(descriptionReference.getReference().getType().getId(), config.getDataRepositoryTypeId()))
             return null;
         DataRepositoryPublicModel model = new DataRepositoryPublicModel();
         Reference reference = descriptionReference.getReference();

dmp-backend/core/src/main/java/eu/eudat/model/publicapi/datasetwizard/ExternalDatasetPublicListingModel.java

@@ -1,11 +1,12 @@
 package eu.eudat.model.publicapi.datasetwizard;
 
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.model.DescriptionReference;
 import eu.eudat.model.Reference;
 import eu.eudat.model.referencedefinition.Field;
+import eu.eudat.service.publicapi.PublicApiProperties;
 
 import java.util.Date;
+import java.util.Objects;
 import java.util.UUID;
 
 public class ExternalDatasetPublicListingModel {
@@ -106,8 +107,8 @@ public class ExternalDatasetPublicListingModel {
         this.source = source;
     }
 
-    public static ExternalDatasetPublicListingModel fromDescriptionReference(DescriptionReference descriptionReference) {
+    public static ExternalDatasetPublicListingModel fromDescriptionReference(DescriptionReference descriptionReference, PublicApiProperties.ReferenceTypeMapConfig config) {
-        if (descriptionReference.getReference().getType() != ReferenceType.Datasets)
+        if (!Objects.equals(descriptionReference.getReference().getType().getId(), config.getDatasetTypeId()))
             return null;
         ExternalDatasetPublicListingModel model = new ExternalDatasetPublicListingModel();
         Reference reference = descriptionReference.getReference();

dmp-backend/core/src/main/java/eu/eudat/model/publicapi/datasetwizard/RegistryPublicModel.java

@@ -1,11 +1,11 @@
 package eu.eudat.model.publicapi.datasetwizard;
 
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.model.DescriptionReference;
 import eu.eudat.model.Reference;
 import eu.eudat.model.referencedefinition.Field;
+import eu.eudat.service.publicapi.PublicApiProperties;
 
-import java.util.Date;
+import java.util.Objects;
 import java.util.UUID;
 
 public class RegistryPublicModel {
@@ -74,8 +74,8 @@ public class RegistryPublicModel {
         return null;
     }
 
-    public static RegistryPublicModel fromDescriptionReference(DescriptionReference descriptionReference) {
+    public static RegistryPublicModel fromDescriptionReference(DescriptionReference descriptionReference, PublicApiProperties.ReferenceTypeMapConfig config) {
-        if (descriptionReference.getReference().getType() != ReferenceType.Registries)
+        if (!Objects.equals(descriptionReference.getReference().getType().getId(), config.getRegistryTypeId()))
             return null;
         RegistryPublicModel model = new RegistryPublicModel();
         Reference reference = descriptionReference.getReference();

dmp-backend/core/src/main/java/eu/eudat/model/publicapi/datasetwizard/ServicePublicModel.java

@@ -1,11 +1,12 @@
 package eu.eudat.model.publicapi.datasetwizard;
 
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.model.DescriptionReference;
 import eu.eudat.model.Reference;
 import eu.eudat.model.referencedefinition.Field;
+import eu.eudat.service.publicapi.PublicApiProperties;
 
 import java.util.Date;
+import java.util.Objects;
 import java.util.UUID;
 
 public class ServicePublicModel {
@@ -74,8 +75,8 @@ public class ServicePublicModel {
         return null;
     }
 
-    public static ServicePublicModel fromDescriptionReference(DescriptionReference descriptionReference) {
+    public static ServicePublicModel fromDescriptionReference(DescriptionReference descriptionReference, PublicApiProperties.ReferenceTypeMapConfig config) {
-        if (descriptionReference.getReference().getType() != ReferenceType.Services)
+        if (!Objects.equals(descriptionReference.getReference().getType().getId(), config.getServiceTypeId()))
             return null;
         ServicePublicModel model = new ServicePublicModel();
         Reference reference = descriptionReference.getReference();

dmp-backend/core/src/main/java/eu/eudat/model/publicapi/grant/GrantPublicOverviewModel.java

@@ -1,15 +1,16 @@
 package eu.eudat.model.publicapi.grant;
 
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.model.DescriptionReference;
 import eu.eudat.model.DmpReference;
 import eu.eudat.model.Reference;
 import eu.eudat.model.publicapi.funder.FunderPublicOverviewModel;
 import eu.eudat.model.referencedefinition.Field;
+import eu.eudat.service.publicapi.PublicApiProperties;
 
 import java.time.Instant;
 import java.util.Date;
 import java.util.List;
+import java.util.Objects;
 import java.util.UUID;
 
 public class GrantPublicOverviewModel {
@@ -78,17 +79,17 @@ public class GrantPublicOverviewModel {
         this.funder = funder;
     }
 
-    public static GrantPublicOverviewModel fromDmpReferences(List<DmpReference> references) {
+    public static GrantPublicOverviewModel fromDmpReferences(List<DmpReference> references, PublicApiProperties.ReferenceTypeMapConfig config) {
         FunderPublicOverviewModel funder = null;
         for (DmpReference dmpReference : references) {
-            if (dmpReference.getReference().getType() == ReferenceType.Funder) {
+            if (!Objects.equals(dmpReference.getReference().getType().getId(), config.getFunderTypeId())) {
                 funder = new FunderPublicOverviewModel();
                 Reference reference = dmpReference.getReference();
                 funder.setId(reference.getId());
                 funder.setLabel(reference.getLabel());
                 continue;
             }
-            if (dmpReference.getReference().getType() != ReferenceType.Grants)
+            if (!Objects.equals(dmpReference.getReference().getType().getId(), config.getGrantTypeId()))
                 continue;
             GrantPublicOverviewModel model = new GrantPublicOverviewModel();
             Reference reference = dmpReference.getReference();
@@ -110,17 +111,17 @@ public class GrantPublicOverviewModel {
         return null;
     }
 
-    public static GrantPublicOverviewModel fromDescriptionReference(List<DescriptionReference> references) {
+    public static GrantPublicOverviewModel fromDescriptionReference(List<DescriptionReference> references, PublicApiProperties.ReferenceTypeMapConfig config) {
         FunderPublicOverviewModel funder = null;
         for (DescriptionReference descriptionReference : references) {
-            if (descriptionReference.getReference().getType() == ReferenceType.Funder) {
+            if (Objects.equals(descriptionReference.getReference().getType().getId(), config.getFunderTypeId())) {
                 funder = new FunderPublicOverviewModel();
                 Reference reference = descriptionReference.getReference();
                 funder.setId(reference.getId());
                 funder.setLabel(reference.getLabel());
                 continue;
             }
-            if (descriptionReference.getReference().getType() != ReferenceType.Grants)
+            if (Objects.equals(descriptionReference.getReference().getType().getId(), config.getGrantTypeId()))
                 continue;
             GrantPublicOverviewModel model = new GrantPublicOverviewModel();
             Reference reference = descriptionReference.getReference();

dmp-backend/core/src/main/java/eu/eudat/model/publicapi/organisation/OrganizationPublicModel.java

@@ -1,8 +1,10 @@
 package eu.eudat.model.publicapi.organisation;
 
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.model.DmpReference;
 import eu.eudat.model.Reference;
+import eu.eudat.service.publicapi.PublicApiProperties;
+
+import java.util.Objects;
 
 public class OrganizationPublicModel {
     private String label;
@@ -46,8 +48,8 @@ public class OrganizationPublicModel {
         this.key = key;
     }
 
-    public static OrganizationPublicModel fromDmpReference(DmpReference dmpReference) {
+    public static OrganizationPublicModel fromDmpReference(DmpReference dmpReference, PublicApiProperties.ReferenceTypeMapConfig config) {
-        if (dmpReference.getReference().getType() != ReferenceType.Organizations)
+        if (!Objects.equals(dmpReference.getReference().getType().getId(), config.getOrganizationTypeId()))
             return null;
         OrganizationPublicModel model = new OrganizationPublicModel();
         Reference reference = dmpReference.getReference();

dmp-backend/core/src/main/java/eu/eudat/model/publicapi/researcher/ResearcherPublicModel.java

@@ -1,12 +1,14 @@
 package eu.eudat.model.publicapi.researcher;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.model.DmpReference;
 import eu.eudat.model.Reference;
+import eu.eudat.service.publicapi.PublicApiProperties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Objects;
+
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class ResearcherPublicModel {
     private static final Logger logger = LoggerFactory.getLogger(ResearcherPublicModel.class);
@@ -51,8 +53,8 @@ public class ResearcherPublicModel {
         this.key = key;
     }
 
-    public static ResearcherPublicModel fromDmpReference(DmpReference dmpReference) {
+    public static ResearcherPublicModel fromDmpReference(DmpReference dmpReference, PublicApiProperties.ReferenceTypeMapConfig config) {
-        if (dmpReference.getReference().getType() != ReferenceType.Researcher)
+        if (!Objects.equals(dmpReference.getReference().getType().getId(), config.getRegistryTypeId()))
             return null;
         ResearcherPublicModel model = new ResearcherPublicModel();
         Reference reference = dmpReference.getReference();

dmp-backend/core/src/main/java/eu/eudat/query/ReferenceQuery.java

@@ -1,7 +1,6 @@
 package eu.eudat.query;
 
 import eu.eudat.authorization.AuthorizationFlags;
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.commons.enums.IsActive;
 import eu.eudat.commons.enums.ReferenceSourceType;
 import eu.eudat.data.DmpReferenceEntity;
@@ -33,17 +32,15 @@ public class ReferenceQuery extends QueryBase<ReferenceEntity> {
 
     private Collection<ReferenceSourceType> referenceSourceTypes;
 
-    private Collection<ReferenceType> referenceTypes;
+    private Collection<UUID> typeIds;
 
     private Collection<String> references;
     private Collection<String> sources;
 
     private Collection<UUID> excludedIds;
 
-
     private DmpReferenceQuery dmpReferenceQuery;
 
-    private Collection<UUID> referenceTypeIds;
 
     private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
 
@@ -82,18 +79,18 @@ public class ReferenceQuery extends QueryBase<ReferenceEntity> {
         return this;
     }
 
-    public ReferenceQuery types(ReferenceType value) {
+    public ReferenceQuery typeIds(UUID value) {
-        this.referenceTypes = List.of(value);
+        this.typeIds = List.of(value);
         return this;
     }
 
-    public ReferenceQuery types(ReferenceType... value) {
+    public ReferenceQuery typeIds(UUID... value) {
-        this.referenceTypes = Arrays.asList(value);
+        this.typeIds = Arrays.asList(value);
         return this;
     }
 
-    public ReferenceQuery types(Collection<ReferenceType> values) {
+    public ReferenceQuery typeIds(Collection<UUID> values) {
-        this.referenceTypes = values;
+        this.typeIds = values;
         return this;
     }
 
@@ -157,21 +154,6 @@ public class ReferenceQuery extends QueryBase<ReferenceEntity> {
         return this;
     }
 
-    public ReferenceQuery referenceTypeIds(UUID value) {
-        this.referenceTypeIds = List.of(value);
-        return this;
-    }
-
-    public ReferenceQuery referenceTypeIds(UUID... value) {
-        this.referenceTypeIds = Arrays.asList(value);
-        return this;
-    }
-
-    public ReferenceQuery referenceTypeIds(Collection<UUID> values) {
-        this.referenceTypeIds = values;
-        return this;
-    }
-    
     public ReferenceQuery dmpReferenceSubQuery(DmpReferenceQuery value) {
         this.dmpReferenceQuery = value;
         return this;
@@ -193,7 +175,7 @@ public class ReferenceQuery extends QueryBase<ReferenceEntity> {
 
     @Override
     protected Boolean isFalseQuery() {
-        return this.isEmpty(this.ids) || this.isEmpty(this.isActives) || this.isEmpty(this.sources) ||this.isEmpty(this.excludedIds) || this.isEmpty(this.referenceTypes) || this.isEmpty(this.referenceSourceTypes)|| this.isFalseQuery(this.dmpReferenceQuery);
+        return this.isEmpty(this.ids) || this.isEmpty(this.isActives) || this.isEmpty(this.sources) ||this.isEmpty(this.excludedIds) || this.isEmpty(this.typeIds) || this.isEmpty(this.referenceSourceTypes)|| this.isFalseQuery(this.dmpReferenceQuery);
     }
 
     @Override
@@ -223,9 +205,9 @@ public class ReferenceQuery extends QueryBase<ReferenceEntity> {
                 inClause.value(item);
             predicates.add(inClause);
         }
-        if (this.referenceTypes != null) {
+        if (this.typeIds != null) {
-            CriteriaBuilder.In<ReferenceType> inClause = queryContext.CriteriaBuilder.in(queryContext.Root.get(ReferenceEntity._type));
+            CriteriaBuilder.In<UUID> inClause = queryContext.CriteriaBuilder.in(queryContext.Root.get(ReferenceEntity._typeId));
-            for (ReferenceType item : this.referenceTypes)
+            for (UUID item : this.typeIds)
                 inClause.value(item);
             predicates.add(inClause);
         }
@@ -273,7 +255,7 @@ public class ReferenceQuery extends QueryBase<ReferenceEntity> {
         item.setReference(QueryBase.convertSafe(tuple, columns, ReferenceEntity._reference, String.class));
         item.setSource(QueryBase.convertSafe(tuple, columns, ReferenceEntity._source, String.class));
         item.setSourceType(QueryBase.convertSafe(tuple, columns, ReferenceEntity._sourceType, ReferenceSourceType.class));
-        item.setType(QueryBase.convertSafe(tuple, columns, ReferenceEntity._type, ReferenceType.class));
+        item.setTypeId(QueryBase.convertSafe(tuple, columns, ReferenceEntity._typeId, UUID.class));
         item.setCreatedById(QueryBase.convertSafe(tuple, columns, ReferenceEntity._createdById, UUID.class));
         return item;
     }
@@ -292,7 +274,8 @@ public class ReferenceQuery extends QueryBase<ReferenceEntity> {
         else if (item.match(Reference._reference) || item.match(PublicReference._reference)) return ReferenceEntity._reference;
         else if (item.match(Reference._source)) return ReferenceEntity._source;
         else if (item.match(Reference._sourceType)) return ReferenceEntity._sourceType;
-        else if (item.match(Reference._type) || item.match(PublicReference._type)) return ReferenceEntity._type;
+        else if (item.match(Reference._type) || item.match(PublicReference._type)) return ReferenceEntity._typeId;
+        else if (item.prefix(Reference._type) || item.prefix(PublicReference._type)) return ReferenceEntity._typeId;
         else if (item.prefix(Reference._createdBy)) return ReferenceEntity._createdById;
         else return null;
     }

dmp-backend/core/src/main/java/eu/eudat/query/lookup/ReferenceDefinitionSearchLookup.java

@@ -40,7 +40,7 @@ public class ReferenceDefinitionSearchLookup extends Lookup {
 
     public ReferenceQuery enrich(QueryFactory queryFactory) {
         ReferenceQuery query = queryFactory.query(ReferenceQuery.class);
-        if (this.referenceTypeId != null) query.referenceTypeIds(this.referenceTypeId);
+        if (this.referenceTypeId != null) query.typeIds(this.referenceTypeId);
 
         this.enrichCommon(query);
 

dmp-backend/core/src/main/java/eu/eudat/query/lookup/ReferenceLookup.java

@@ -1,6 +1,5 @@
 package eu.eudat.query.lookup;
 
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.commons.enums.IsActive;
 import eu.eudat.commons.enums.ReferenceSourceType;
 import eu.eudat.query.ReferenceQuery;
@@ -19,7 +18,7 @@ public class ReferenceLookup extends Lookup {
 
     private Collection<ReferenceSourceType> sourceTypes;
 
-    private Collection<ReferenceType> types;
+    private Collection<UUID> typeIds;
 
     private List<UUID> ids;
 
@@ -65,19 +64,19 @@ public class ReferenceLookup extends Lookup {
         this.sourceTypes = sourceTypes;
     }
 
-    public Collection<ReferenceType> getTypes() {
+    public Collection<UUID> getTypeIds() {
-        return types;
+        return typeIds;
     }
 
-    public void setTypes(Collection<ReferenceType> types) {
+    public void setTypeIds(Collection<UUID> typeIds) {
-        this.types = types;
+        this.typeIds = typeIds;
     }
 
     public ReferenceQuery enrich(QueryFactory queryFactory) {
         ReferenceQuery query = queryFactory.query(ReferenceQuery.class);
         if (this.like != null) query.like(this.like);
         if (this.isActive != null) query.isActive(this.isActive);
-        if (this.types != null) query.types(this.types);
+        if (this.typeIds != null) query.typeIds(this.typeIds);
         if (this.sourceTypes != null) query.sourceTypes(this.sourceTypes);
         if (this.ids != null) query.ids(this.ids);
         if (this.excludedIds != null) query.excludedIds(this.excludedIds);

dmp-backend/core/src/main/java/eu/eudat/query/lookup/ReferenceSearchLookup.java

@@ -1,15 +1,16 @@
 package eu.eudat.query.lookup;
 
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.query.ReferenceQuery;
 import gr.cite.tools.data.query.Lookup;
 import gr.cite.tools.data.query.QueryFactory;
 
+import java.util.UUID;
+
 public class ReferenceSearchLookup extends Lookup {
 
     private String like;
 
-    private ReferenceType type;
+    private UUID typeId;
 
     private String key;
 
@@ -21,12 +22,12 @@ public class ReferenceSearchLookup extends Lookup {
         this.like = like;
     }
 
-    public ReferenceType getType() {
+    public UUID getTypeId() {
-        return type;
+        return typeId;
     }
 
-    public void setType(ReferenceType type) {
+    public void setTypeId(UUID typeId) {
-        this.type = type;
+        this.typeId = typeId;
     }
 
     public String getKey() {
@@ -39,7 +40,7 @@ public class ReferenceSearchLookup extends Lookup {
 
     public ReferenceQuery enrich(QueryFactory queryFactory) {
         ReferenceQuery query = queryFactory.query(ReferenceQuery.class);
-        if (this.type != null) query.types(this.type);
+        if (this.typeId != null) query.typeIds(this.typeId);
 
         this.enrichCommon(query);
 

dmp-backend/core/src/main/java/eu/eudat/query/lookup/ReferenceTypeLookup.java

@@ -1,14 +1,10 @@
 package eu.eudat.query.lookup;
 
 import eu.eudat.commons.enums.IsActive;
-import eu.eudat.commons.enums.ReferenceSourceType;
-import eu.eudat.commons.enums.ReferenceType;
-import eu.eudat.query.ReferenceQuery;
 import eu.eudat.query.ReferenceTypeQuery;
 import gr.cite.tools.data.query.Lookup;
 import gr.cite.tools.data.query.QueryFactory;
 
-import java.util.Collection;
 import java.util.List;
 import java.util.UUID;
 

dmp-backend/core/src/main/java/eu/eudat/service/dashborad/DashboardServiceConfiguration.java

@@ -0,0 +1,10 @@
+package eu.eudat.service.dashborad;
+
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@EnableConfigurationProperties(DashboardServiceProperties.class)
+public class DashboardServiceConfiguration {
+
+}

dmp-backend/core/src/main/java/eu/eudat/service/dashborad/DashboardServiceImpl.java

@@ -6,13 +6,13 @@ import eu.eudat.authorization.Permission;
 import eu.eudat.commons.JsonHandlingService;
 import eu.eudat.commons.XmlHandlingService;
 import eu.eudat.commons.enums.*;
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.commons.scope.user.UserScope;
 import eu.eudat.commons.types.dashborad.RecentActivityItemEntity;
 import eu.eudat.convention.ConventionService;
 import eu.eudat.errorcode.ErrorThesaurusProperties;
 import eu.eudat.event.EventBroker;
 import eu.eudat.model.*;
+import eu.eudat.model.builder.PublicReferenceTypeBuilder;
 import eu.eudat.model.builder.RecentActivityItemBuilder;
 import eu.eudat.model.result.QueryResult;
 import eu.eudat.query.*;
@@ -43,58 +43,27 @@ public class DashboardServiceImpl implements DashboardService {
 
     private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DashboardServiceImpl.class));
 
-    private final EntityManager entityManager;
-
-    private final AuthorizationService authorizationService;
-
-    private final DeleterFactory deleterFactory;
-
-    private final BuilderFactory builderFactory;
-
     private final ConventionService conventionService;
-
+    private final AuthorizationService authorizationService;
-    private final ErrorThesaurusProperties errors;
+    private final BuilderFactory builderFactory;
-
-    private final MessageSource messageSource;
-
-    private final EventBroker eventBroker;
-
     private final QueryFactory queryFactory;
-
-    private final JsonHandlingService jsonHandlingService;
-
     private final UserScope userScope;
-    private final XmlHandlingService xmlHandlingService;
+    private final DashboardServiceProperties config; 
-
     private final ElasticQueryHelperService elasticQueryHelperService;
-    
     private final DashboardStatisticsCacheService dashboardStatisticsCacheService;
     @Autowired
     public DashboardServiceImpl(
-            EntityManager entityManager,
+		    ConventionService conventionService, AuthorizationService authorizationService,
-            AuthorizationService authorizationService,
-            DeleterFactory deleterFactory,
 		    BuilderFactory builderFactory,
-            ConventionService conventionService,
-            ErrorThesaurusProperties errors,
-            MessageSource messageSource,
-            EventBroker eventBroker,
 		    QueryFactory queryFactory,
-            JsonHandlingService jsonHandlingService,
 		    UserScope userScope,
-            XmlHandlingService xmlHandlingService, ElasticQueryHelperService elasticQueryHelperService, DashboardStatisticsCacheService dashboardStatisticsCacheService) {
+		    DashboardServiceProperties config, ElasticQueryHelperService elasticQueryHelperService, DashboardStatisticsCacheService dashboardStatisticsCacheService) {
-        this.entityManager = entityManager;
-        this.authorizationService = authorizationService;
-        this.deleterFactory = deleterFactory;
-        this.builderFactory = builderFactory;
 	    this.conventionService = conventionService;
-        this.errors = errors;
+	    this.authorizationService = authorizationService;
-        this.messageSource = messageSource;
+        this.builderFactory = builderFactory;
-        this.eventBroker = eventBroker;
         this.queryFactory = queryFactory;
-        this.jsonHandlingService = jsonHandlingService;
         this.userScope = userScope;
-        this.xmlHandlingService = xmlHandlingService;
+	    this.config = config;
 	    this.elasticQueryHelperService = elasticQueryHelperService;
         this.dashboardStatisticsCacheService = dashboardStatisticsCacheService;
     }
@@ -146,12 +115,19 @@ public class DashboardServiceImpl implements DashboardService {
             DashboardStatistics statistics = new DashboardStatistics();
             statistics.setDmpCount(dmpQuery.authorize(EnumSet.of(Public)).count());
             statistics.setDescriptionCount(this.queryFactory.query(DescriptionQuery.class).isActive(IsActive.Active).dmpSubQuery(dmpQuery).statuses(DescriptionStatus.Finalized).authorize(EnumSet.of(Public)).count());
-            statistics.setOrganizationCount(this.queryFactory.query(ReferenceQuery.class).isActive(IsActive.Active).types(ReferenceType.Organizations).authorize(EnumSet.of(Public))
+
-                    .dmpReferenceSubQuery(this.queryFactory.query(DmpReferenceQuery.class).isActives(IsActive.Active)
+            statistics.setReferenceTypeStatistics(new ArrayList<>());
-                            .dmpSubQuery(dmpQuery)).count());
+            if (!this.conventionService.isListNullOrEmpty(this.config.getReferenceTypeCounters())){
-            statistics.setGrantCount(this.queryFactory.query(ReferenceQuery.class).isActive(IsActive.Active).types(ReferenceType.Grants).authorize(EnumSet.of(Public))
+                for (UUID typeId : this.config.getReferenceTypeCounters()){
+                    DashboardReferenceTypeStatistics referenceTypeStatistics = new DashboardReferenceTypeStatistics();
+                    referenceTypeStatistics.setCount(this.queryFactory.query(ReferenceQuery.class).isActive(IsActive.Active).typeIds(typeId).authorize(EnumSet.of(Public))
                             .dmpReferenceSubQuery(this.queryFactory.query(DmpReferenceQuery.class).isActives(IsActive.Active)
                                     .dmpSubQuery(dmpQuery)).count());
+                    referenceTypeStatistics.setReferenceType(this.builderFactory.builder(PublicReferenceTypeBuilder.class).build(new BaseFieldSet().ensure(PublicReferenceType._name), this.queryFactory.query(ReferenceTypeQuery.class).ids(typeId).first()));
+                    statistics.getReferenceTypeStatistics().add(referenceTypeStatistics);
+                }
+            }
+            
             cacheValue = new DashboardStatisticsCacheService.DashboardStatisticsCacheValue();
             cacheValue.setPublic(true);
             cacheValue.setDashboardStatistics(statistics);
@@ -175,13 +151,18 @@ public class DashboardServiceImpl implements DashboardService {
             DashboardStatistics statistics = new DashboardStatistics();
             statistics.setDmpCount(dmpQuery.authorize(EnumSet.of(AuthorizationFlags.DmpAssociated)).count());
             statistics.setDescriptionCount(this.queryFactory.query(DescriptionQuery.class).isActive(IsActive.Active).dmpSubQuery(dmpQuery).statuses(DescriptionStatus.Finalized).authorize(EnumSet.of(AuthorizationFlags.DmpAssociated)).count());
-            statistics.setOrganizationCount(this.queryFactory.query(ReferenceQuery.class).isActive(IsActive.Active).types(ReferenceType.Organizations).authorize(EnumSet.of(Owner))
-                    .dmpReferenceSubQuery(this.queryFactory.query(DmpReferenceQuery.class).isActives(IsActive.Active)
-                            .dmpSubQuery(dmpQuery)).count());
-            statistics.setGrantCount(this.queryFactory.query(ReferenceQuery.class).isActive(IsActive.Active).types(ReferenceType.Grants).authorize(EnumSet.of(Owner))
-                    .dmpReferenceSubQuery(this.queryFactory.query(DmpReferenceQuery.class).isActives(IsActive.Active)
-                            .dmpSubQuery(dmpQuery)).count());
 
+            statistics.setReferenceTypeStatistics(new ArrayList<>());
+            if (!this.conventionService.isListNullOrEmpty(this.config.getReferenceTypeCounters())){
+                for (UUID typeId : this.config.getReferenceTypeCounters()){
+                    DashboardReferenceTypeStatistics referenceTypeStatistics = new DashboardReferenceTypeStatistics();
+                    referenceTypeStatistics.setCount(this.queryFactory.query(ReferenceQuery.class).isActive(IsActive.Active).typeIds(typeId).authorize(EnumSet.of(Owner))
+                            .dmpReferenceSubQuery(this.queryFactory.query(DmpReferenceQuery.class).isActives(IsActive.Active)
+                                    .dmpSubQuery(dmpQuery)).count());
+                    referenceTypeStatistics.setReferenceType(this.builderFactory.builder(PublicReferenceTypeBuilder.class).build(new BaseFieldSet().ensure(PublicReferenceType._name), this.queryFactory.query(ReferenceTypeQuery.class).ids(typeId).first()));
+                    statistics.getReferenceTypeStatistics().add(referenceTypeStatistics);
+                }
+            }
             cacheValue = new DashboardStatisticsCacheService.DashboardStatisticsCacheValue(this.userScope.getUserId());
             cacheValue.setPublic(false);
             cacheValue.setDashboardStatistics(statistics);

dmp-backend/core/src/main/java/eu/eudat/service/dashborad/DashboardServiceProperties.java

@@ -0,0 +1,21 @@
+package eu.eudat.service.dashborad;
+
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import java.util.List;
+import java.util.UUID;
+
+@ConfigurationProperties(prefix = "dashboard")
+public class DashboardServiceProperties {
+	private List<UUID> referenceTypeCounters;
+
+	public List<UUID> getReferenceTypeCounters() {
+		return referenceTypeCounters;
+	}
+
+	public void setReferenceTypeCounters(List<UUID> referenceTypeCounters) {
+		this.referenceTypeCounters = referenceTypeCounters;
+	}
+}
+

dmp-backend/core/src/main/java/eu/eudat/service/description/DescriptionServiceImpl.java

@@ -5,12 +5,12 @@ import eu.eudat.authorization.Permission;
 import eu.eudat.commons.JsonHandlingService;
 import eu.eudat.commons.XmlHandlingService;
 import eu.eudat.commons.enums.*;
-import eu.eudat.commons.enums.ReferenceType;
 import eu.eudat.commons.enums.notification.NotificationContactType;
 import eu.eudat.commons.scope.user.UserScope;
 import eu.eudat.commons.types.description.*;
 import eu.eudat.commons.types.descriptionreference.DescriptionReferenceDataEntity;
 import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity;
+import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity;
 import eu.eudat.commons.types.descriptiontemplate.fielddata.UploadDataEntity;
 import eu.eudat.commons.types.notification.*;
 import eu.eudat.commons.types.reference.DefinitionEntity;
@@ -184,7 +184,7 @@ public class DescriptionServiceImpl implements DescriptionService {
         this.entityManager.flush();
 
         eu.eudat.commons.types.descriptiontemplate.DefinitionEntity definition =  this.xmlHandlingService.fromXmlSafe(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition());
-        Map<String, List<UUID>> fieldToReferenceMap = this.patchAndSaveReferences(this.buildDescriptionReferencePersists(model.getProperties()), data.getId());
+        Map<String, List<UUID>> fieldToReferenceMap = this.patchAndSaveReferences(this.buildDescriptionReferencePersists(model.getProperties()), data.getId(), definition);
 
         this.entityManager.flush();
         
@@ -424,12 +424,6 @@ public class DescriptionServiceImpl implements DescriptionService {
                 for (UUID id : ids){
                     if (!existingIds.contains(id)) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, Description.class.getSimpleName()}, LocaleContextHolder.getLocale()));
                 }
-            } if (FieldType.INTERNAL_DMP_ENTRIES_RESEARCHERS.equals(fieldType) && !this.conventionService.isListNullOrEmpty(persist.getTextListValue())){
-                List<UUID> ids = persist.getTextListValue().stream().map(UUID::fromString).toList();
-                Set<UUID> existingIds = this.queryFactory.query(ReferenceQuery.class).ids(ids).types(ReferenceType.Researcher).isActive(IsActive.Active).collectAs(new BaseFieldSet().ensure(Reference._id)).stream().map(ReferenceEntity::getId).collect(Collectors.toSet());
-                for (UUID id : ids){
-                    if (!existingIds.contains(id)) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, Reference.class.getSimpleName()}, LocaleContextHolder.getLocale()));
-                }
             }
             data.setTextListValue(persist.getTextListValue());
         }
@@ -483,7 +477,7 @@ public class DescriptionServiceImpl implements DescriptionService {
         }
     }
 
-    private Map<String, List<UUID>> patchAndSaveReferences(List<DescriptionReferencePersist> models, UUID descriptionId) throws InvalidApplicationException {
+    private Map<String, List<UUID>> patchAndSaveReferences(List<DescriptionReferencePersist> models, UUID descriptionId, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity definition) throws InvalidApplicationException {
         if (models == null) models = new ArrayList<>();
         
         Map<String, List<UUID>> fieldToReferenceMap = new HashMap<>();
@@ -512,13 +506,16 @@ public class DescriptionServiceImpl implements DescriptionService {
                 referenceEntity = this.entityManager.find(ReferenceEntity.class, referencePersist.getId());
                 if (referenceEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{referencePersist.getId(), Reference.class.getSimpleName()}, LocaleContextHolder.getLocale()));
             } else {
-                referenceEntity = this.queryFactory.query(ReferenceQuery.class).sourceTypes(referencePersist.getSourceType()).sources(referencePersist.getSource()).isActive(IsActive.Active).references(referencePersist.getReference()).first();
+                ReferenceTypeDataEntity fieldEntity = definition.getFieldById(model.getData().getFieldId()).stream().filter(x-> x.getData() != null && x.getData().getFieldType().equals(FieldType.REFERENCE_TYPES)).map(x-> (ReferenceTypeDataEntity)x.getData()).findFirst().orElse(null);
+                if (fieldEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getData().getFieldId(), ReferenceTypeDataEntity.class.getSimpleName()}, LocaleContextHolder.getLocale()));
+
+                referenceEntity = this.queryFactory.query(ReferenceQuery.class).sourceTypes(referencePersist.getSourceType()).typeIds(fieldEntity.getReferenceTypeId()).sources(referencePersist.getSource()).isActive(IsActive.Active).references(referencePersist.getReference()).first();
                 if (referenceEntity == null){
                     referenceEntity = new ReferenceEntity();
                     referenceEntity.setId(UUID.randomUUID());
                     referenceEntity.setIsActive(IsActive.Active);
                     referenceEntity.setCreatedAt(Instant.now());
-                    referenceEntity.setType(referencePersist.getType());
+                    referenceEntity.setTypeId(fieldEntity.getReferenceTypeId());
 
                     referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(this.buildDefinitionEntity(referencePersist.getDefinition())));
                     referenceEntity.setUpdatedAt(Instant.now());

dmp-backend/core/src/main/java/eu/eudat/service/dmp/DmpServiceImpl.java

@@ -12,6 +12,7 @@ import eu.eudat.commons.types.actionconfirmation.DmpInvitationEntity;
 import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity;
 import eu.eudat.commons.types.dmp.DmpContactEntity;
 import eu.eudat.commons.types.dmp.DmpPropertiesEntity;
+import eu.eudat.commons.types.dmpblueprint.ReferenceTypeFieldEntity;
 import eu.eudat.commons.types.dmpreference.DmpReferenceDataEntity;
 import eu.eudat.commons.types.notification.*;
 import eu.eudat.commons.types.reference.DefinitionEntity;
@@ -24,10 +25,7 @@ import eu.eudat.event.DmpTouchedEvent;
 import eu.eudat.event.EventBroker;
 import eu.eudat.integrationevent.outbox.notification.NotifyIntegrationEvent;
 import eu.eudat.integrationevent.outbox.notification.NotifyIntegrationEventHandler;
-import eu.eudat.model.Dmp;
+import eu.eudat.model.*;
-import eu.eudat.model.DmpUser;
-import eu.eudat.model.Reference;
-import eu.eudat.model.UserContactInfo;
 import eu.eudat.model.builder.DmpBuilder;
 import eu.eudat.model.builder.DmpUserBuilder;
 import eu.eudat.model.deleter.*;
@@ -169,7 +167,12 @@ public class DmpServiceImpl implements DmpService {
 
         DmpEntity data = this.patchAndSave(model);
 
-        this.patchAndSaveReferences(this.buildDmpReferencePersists(model.getProperties()), data.getId());
+        DmpBlueprintEntity blueprintEntity = this.entityManager.find(DmpBlueprintEntity.class, data.getBlueprintId());
+        if (blueprintEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{data.getBlueprintId(), DmpBlueprint.class.getSimpleName()}, LocaleContextHolder.getLocale()));
+        eu.eudat.commons.types.dmpblueprint.DefinitionEntity definition = this.xmlHandlingService.fromXmlSafe(eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class, blueprintEntity.getDefinition());
+        if (definition == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{data.getBlueprintId(), eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class.getSimpleName()}, LocaleContextHolder.getLocale()));
+        
+        this.patchAndSaveReferences(this.buildDmpReferencePersists(model.getProperties()), data.getId(), definition);
 
         this.patchAndSaveTemplates(data.getId(), model.getDescriptionTemplates());
 
@@ -620,7 +623,7 @@ public class DmpServiceImpl implements DmpService {
         return dmpReferencePersists;
     }
 
-    private void patchAndSaveReferences(List<DmpReferencePersist> models, UUID dmpId) throws InvalidApplicationException {
+    private void patchAndSaveReferences(List<DmpReferencePersist> models, UUID dmpId, eu.eudat.commons.types.dmpblueprint.DefinitionEntity blueprintDefinition) throws InvalidApplicationException {
         if (models == null) models = new ArrayList<>();
 
         List<DmpReferenceEntity> dmpReferences = this.queryFactory.query(DmpReferenceQuery.class).dmpIds(dmpId).collect();
@@ -647,13 +650,16 @@ public class DmpServiceImpl implements DmpService {
                 referenceEntity = this.entityManager.find(ReferenceEntity.class, referencePersist.getId());
                 if (referenceEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{referencePersist.getId(), Reference.class.getSimpleName()}, LocaleContextHolder.getLocale()));
             } else {
-                referenceEntity = this.queryFactory.query(ReferenceQuery.class).sourceTypes(referencePersist.getSourceType()).sources(referencePersist.getSource()).isActive(IsActive.Active).references(referencePersist.getReference()).first();
+                ReferenceTypeFieldEntity fieldEntity = blueprintDefinition.getFieldById(model.getData().getBlueprintFieldId()).stream().filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.ReferenceType)).map(x-> (ReferenceTypeFieldEntity)x).findFirst().orElse(null);
+                if (fieldEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getData().getBlueprintFieldId(), ReferenceTypeFieldEntity.class.getSimpleName()}, LocaleContextHolder.getLocale()));
+                
+                referenceEntity = this.queryFactory.query(ReferenceQuery.class).sourceTypes(referencePersist.getSourceType()).typeIds(fieldEntity.getReferenceTypeId()).sources(referencePersist.getSource()).isActive(IsActive.Active).references(referencePersist.getReference()).first();
                 if (referenceEntity == null){
                     referenceEntity = new ReferenceEntity();
                     referenceEntity.setId(UUID.randomUUID());
                     referenceEntity.setIsActive(IsActive.Active);
                     referenceEntity.setCreatedAt(Instant.now());
-                    referenceEntity.setType(referencePersist.getType());
+                    referenceEntity.setTypeId(fieldEntity.getReferenceTypeId());
 
                     referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(this.buildDefinitionEntity(referencePersist.getDefinition())));
                     referenceEntity.setUpdatedAt(Instant.now());

dmp-backend/core/src/main/java/eu/eudat/service/dmpblueprint/DmpBlueprintServiceImpl.java

@@ -19,7 +19,6 @@ import eu.eudat.model.dmpblueprintdefinition.Section;
 import eu.eudat.model.persist.DmpBlueprintPersist;
 import eu.eudat.model.persist.NewVersionDmpBlueprintPersist;
 import eu.eudat.model.persist.dmpblueprintdefinition.*;
-import eu.eudat.query.DescriptionTemplateQuery;
 import eu.eudat.query.DmpBlueprintQuery;
 import eu.eudat.service.responseutils.ResponseUtilsService;
 import gr.cite.commons.web.authz.service.AuthorizationService;
@@ -37,7 +36,6 @@ import gr.cite.tools.logging.MapLogEntry;
 import gr.cite.tools.validation.ValidatorFactory;
 import jakarta.persistence.EntityManager;
 import jakarta.xml.bind.JAXBException;
-import org.apache.bcel.generic.DADD;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -252,15 +250,26 @@ public class DmpBlueprintServiceImpl implements DmpBlueprintService {
         if (persist == null)
             return new ExtraFieldEntity();
         FieldEntity data;
-        if (DmpBlueprintFieldCategory.Extra.equals(persist.getCategory())) {
+        
+        switch (persist.getCategory()){
+            case Extra -> {
                 ExtraFieldEntity dataTyped = new ExtraFieldEntity();
                 dataTyped.setType(((ExtraFieldPersist) persist).getDataType());
                 data = dataTyped;
-        } else {
+            }
+            case System -> {
                 SystemFieldEntity dataTyped = new SystemFieldEntity();
                 dataTyped.setType(((SystemFieldPersist) persist).getSystemFieldType());
                 data = dataTyped;
             }
+            case ReferenceType -> {
+                ReferenceTypeFieldEntity dataTyped = new ReferenceTypeFieldEntity();
+                dataTyped.setReferenceTypeId(((ReferenceTypeFieldPersist) persist).getReferenceTypeId());
+                data = dataTyped;
+            }
+            default -> throw new  InternalError("unknown type: " + persist.getCategory());
+        }
+        
         data.setId(persist.getId());
         data.setCategory(persist.getCategory());
         data.setLabel(persist.getLabel());
@@ -472,6 +481,7 @@ public class DmpBlueprintServiceImpl implements DmpBlueprintService {
             }
         }
         xml.setSystemFields(dmpBlueprintSystemFieldModels);
+        
         List<ExtraFieldImportExport> dmpBlueprintExtraFieldModels = new LinkedList<>();
         if (!this.conventionService.isListNullOrEmpty(entity.getFields())) {
             for (ExtraFieldEntity systemField : entity.getFields().stream().filter(x -> x.getCategory() == DmpBlueprintFieldCategory.Extra).map(x -> (ExtraFieldEntity) x).toList()) {
@@ -480,6 +490,14 @@ public class DmpBlueprintServiceImpl implements DmpBlueprintService {
         }
         xml.setExtraFields(dmpBlueprintExtraFieldModels);
 
+        List<ReferenceTypeFieldImportExport> dmpBlueprintReferenceFieldModels = new LinkedList<>();
+        if (!this.conventionService.isListNullOrEmpty(entity.getFields())) {
+            for (ReferenceTypeFieldEntity referenceTypeFieldEntity : entity.getFields().stream().filter(x -> x.getCategory() == DmpBlueprintFieldCategory.ReferenceType).map(x -> (ReferenceTypeFieldEntity) x).toList()) {
+                dmpBlueprintReferenceFieldModels.add(this.referenceFieldXmlToExport(referenceTypeFieldEntity));
+            }
+        }
+        xml.setReferenceFields(dmpBlueprintReferenceFieldModels);
+
         List<DescriptionTemplateImportExport> dmpBlueprintDescriptionTemplates = new LinkedList<>();
         if (!this.conventionService.isListNullOrEmpty(entity.getDescriptionTemplates())) {
             for (DescriptionTemplateEntity descriptionTemplate : entity.getDescriptionTemplates()) {
@@ -511,6 +529,18 @@ public class DmpBlueprintServiceImpl implements DmpBlueprintService {
         return xml;
     }
 
+    private ReferenceTypeFieldImportExport referenceFieldXmlToExport(ReferenceTypeFieldEntity entity) {
+        ReferenceTypeFieldImportExport xml = new ReferenceTypeFieldImportExport();
+        xml.setId(entity.getId());
+        xml.setReferenceTypeId(entity.getReferenceTypeId());
+        xml.setLabel(entity.getLabel());
+        xml.setPlaceholder(entity.getPlaceholder());
+        xml.setDescription(entity.getDescription());
+        xml.setOrdinal(entity.getOrdinal());
+        xml.setRequired(entity.isRequired());
+        return xml;
+    }
+
     private SystemFieldImportExport systemFieldXmlToExport(SystemFieldEntity entity) {
         SystemFieldImportExport xml = new SystemFieldImportExport();
         xml.setId(entity.getId());
@@ -572,6 +602,11 @@ public class DmpBlueprintServiceImpl implements DmpBlueprintService {
                 dmpBlueprintFieldModels.add(this.xmlSystemFieldToPersist(systemField));
             }
         }
+        if (!this.conventionService.isListNullOrEmpty(importXml.getReferenceFields())) {
+            for (ReferenceTypeFieldImportExport referenceField : importXml.getReferenceFields()) {
+                dmpBlueprintFieldModels.add(this.xmlReferenceFieldToPersist(referenceField));
+            }
+        }
         if (!this.conventionService.isListNullOrEmpty(importXml.getExtraFields())) {
             for (ExtraFieldImportExport extraField : importXml.getExtraFields()) {
                 dmpBlueprintFieldModels.add(this.xmlExtraFieldToPersist(extraField));
@@ -623,6 +658,19 @@ public class DmpBlueprintServiceImpl implements DmpBlueprintService {
         return persist;
     }
 
+    private FieldPersist xmlReferenceFieldToPersist(ReferenceTypeFieldImportExport importXml) {
+        ReferenceTypeFieldPersist persist = new ReferenceTypeFieldPersist();
+        persist.setId(importXml.getId());
+        persist.setCategory(DmpBlueprintFieldCategory.ReferenceType);
+        persist.setReferenceTypeId(importXml.getReferenceTypeId());
+        persist.setLabel(importXml.getLabel());
+        persist.setPlaceholder(importXml.getPlaceholder());
+        persist.setDescription(importXml.getDescription());
+        persist.setOrdinal(importXml.getOrdinal());
+        persist.setRequired(importXml.isRequired());
+        return persist;
+    }
+
     //endregion
 }
 

dmp-backend/core/src/main/java/eu/eudat/service/fielddatahelper/ExternalSelectFieldDataHelperService.java

@@ -1,236 +0,0 @@
-package eu.eudat.service.fielddatahelper;
-
-import eu.eudat.authorization.AuthorizationFlags;
-import eu.eudat.commons.types.descriptiontemplate.fielddata.ExternalSelectDataEntity;
-import eu.eudat.commons.types.descriptiontemplate.importexport.fielddata.*;
-import eu.eudat.convention.ConventionService;
-import eu.eudat.model.builder.descriptiontemplatedefinition.fielddata.ExternalSelectDataBuilder;
-import eu.eudat.model.descriptiontemplatedefinition.fielddata.ExternalSelectData;
-import eu.eudat.model.persist.descriptiontemplatedefinition.fielddata.*;
-import gr.cite.tools.data.builder.BuilderFactory;
-import gr.cite.tools.fieldset.FieldSet;
-import gr.cite.tools.validation.Validator;
-import gr.cite.tools.validation.ValidatorFactory;
-import org.jetbrains.annotations.NotNull;
-import org.springframework.stereotype.Component;
-
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-
-@Component
-public class ExternalSelectFieldDataHelperService extends BaseFieldDataHelperService<ExternalSelectData, ExternalSelectDataPersist, ExternalSelectDataEntity, ExternalSelectImportExport> {
-	private final ConventionService conventionService;
-	private final BuilderFactory builderFactory;
-	private final ValidatorFactory validatorFactory;
-
-	public ExternalSelectFieldDataHelperService(ConventionService conventionService, BuilderFactory builderFactory, ValidatorFactory validatorFactory) {
-		this.conventionService = conventionService;
-		this.builderFactory = builderFactory;
-		this.validatorFactory = validatorFactory;
-	}
-
-
-	@Override
-	public ExternalSelectDataEntity newDataInstanceInternal() {
-		return new ExternalSelectDataEntity();
-	}
-
-	@Override
-	public ExternalSelectData newModelInstanceInternal() {
-		return new ExternalSelectData();
-	}
-
-	@Override
-	public ExternalSelectDataPersist newPersistModelInstanceInternal() {
-		return new ExternalSelectDataPersist();
-	}
-
-	@Override
-	protected ExternalSelectImportExport newImportExportInstanceInternal() {
-		return new ExternalSelectImportExport();
-	}
-
-	@Override
-	public Class<?> getDataClass() {
-		return ExternalSelectDataEntity.class;
-	}
-	@Override
-	public Class<?> getModelClass() {
-		return ExternalSelectData.class;
-	}
-	@Override
-	public Class<?> getPersistModelClass() {
-		return ExternalSelectDataPersist.class;
-	}
-
-	@Override
-	public Class<?> getImportExportClass() {
-		return ExternalSelectImportExport.class;
-	}
-
-	@Override
-	public Validator getPersistModelValidator() {
-		return this.validatorFactory.validator(ExternalSelectDataPersist.AutoCompleteDataPersistValidator.class);
-	}
-
-	@Override
-	public List<ExternalSelectData> buildInternal(FieldSet fieldSet, List<ExternalSelectDataEntity> data, EnumSet<AuthorizationFlags> authorizationFlags){
-		return this.builderFactory.builder(ExternalSelectDataBuilder.class).authorize(authorizationFlags).build(fieldSet, data);
-	}
-
-	@Override
-	protected ExternalSelectDataEntity applyPersistInternal(ExternalSelectDataPersist persist, ExternalSelectDataEntity data) {
-		data.setMultipleSelect(persist.getMultipleSelect());
-		if (!this.conventionService.isListNullOrEmpty(persist.getSources())){
-			data.setSources(new ArrayList<>());
-			for (ExternalSelectDataPersist.ExternalSelectSourcePersist sourcePersist: persist.getSources()) {
-				data.getSources().add(this.buildAutoCompleteSingleData(sourcePersist));
-			}
-		}
-		return data;
-	}
-
-	private @NotNull ExternalSelectDataEntity.ExternalSelectSourceEntity buildAutoCompleteSingleData(ExternalSelectDataPersist.ExternalSelectSourcePersist persist){
-		ExternalSelectDataEntity.ExternalSelectSourceEntity data = new ExternalSelectDataEntity.ExternalSelectSourceEntity();
-		if (persist == null) return data;
-
-		data.setUrl(persist.getUrl());
-		data.setOptionsRoot(persist.getOptionsRoot());
-		data.setHasAuth(persist.getHasAuth());
-		if(persist.getSourceBinding() != null) data.setSourceBinding(this.buildOption(persist.getSourceBinding()));
-		if(persist.getAuth() != null) data.setAuth(this.buildAuthAutoCompleteData(persist.getAuth()));
-		data.setMethod(persist.getMethod());
-		
-		return data;
-	}
-
-	private @NotNull ExternalSelectDataEntity.ExternalSelectSourceBindingEntity buildOption(ExternalSelectDataPersist.ExternalSelectSourceBindingPersist persist){
-		ExternalSelectDataEntity.ExternalSelectSourceBindingEntity data = new ExternalSelectDataEntity.ExternalSelectSourceBindingEntity();
-		if (persist == null) return data;
-
-		data.setLabel(persist.getLabel());
-		data.setValue(persist.getValue());
-		data.setSource(persist.getSource());
-
-		return data;
-	}
-
-	private @NotNull ExternalSelectDataEntity.ExternalSelectAuthDataEntity buildAuthAutoCompleteData(ExternalSelectDataPersist.ExternalSelectAuthDataPersist persist){
-		ExternalSelectDataEntity.ExternalSelectAuthDataEntity data = new ExternalSelectDataEntity.ExternalSelectAuthDataEntity();
-		if (persist == null) return data;
-
-		data.setBody(persist.getBody());
-		data.setUrl(persist.getUrl());
-		data.setMethod(persist.getMethod());
-		data.setPath(persist.getPath());
-		data.setType(persist.getType());
-
-		return data;
-	}
-
-	@Override
-	protected ExternalSelectDataPersist importExportMapDataToPersistInternal(ExternalSelectImportExport data, ExternalSelectDataPersist persist) {
-		persist.setMultipleSelect(data.getMultipleSelect());
-		if (!this.conventionService.isListNullOrEmpty(data.getSources())){
-			persist.setSources(new ArrayList<>());
-			for (ExternalSelectImportExport.ExternalSelectSourceImportExport autoCompleteSingleData: data.getSources()) {
-				persist.getSources().add(this.buildAutoCompleteSingleData(autoCompleteSingleData));
-			}
-		}
-		return persist;
-	}
-
-	private @NotNull ExternalSelectDataPersist.ExternalSelectSourcePersist buildAutoCompleteSingleData(ExternalSelectImportExport.ExternalSelectSourceImportExport data){
-		ExternalSelectDataPersist.ExternalSelectSourcePersist persist = new ExternalSelectDataPersist.ExternalSelectSourcePersist();
-		if (data == null) return persist;
-
-		persist.setUrl(data.getUrl());
-		persist.setOptionsRoot(data.getOptionsRoot());
-		persist.setHasAuth(data.getHasAuth());
-		if(data.getSourceBinding() != null) persist.setSourceBinding(this.buildOption(data.getSourceBinding()));
-		if(data.getAuth() != null) persist.setAuth(this.buildAuthAutoCompleteData(data.getAuth()));
-		persist.setMethod(data.getMethod());
-
-		return persist;
-	}
-
-	private @NotNull ExternalSelectDataPersist.ExternalSelectSourceBindingPersist buildOption(ExternalSelectImportExport.ExternalSelectSourceBindingImportExport data){
-		ExternalSelectDataPersist.ExternalSelectSourceBindingPersist persist = new ExternalSelectDataPersist.ExternalSelectSourceBindingPersist();
-		if (data == null) return persist;
-
-		persist.setLabel(data.getLabel());
-		persist.setValue(data.getValue());
-		persist.setSource(data.getSource());
-
-		return persist;
-	}
-
-	private @NotNull ExternalSelectDataPersist.ExternalSelectAuthDataPersist buildAuthAutoCompleteData(ExternalSelectImportExport.ExternalSelectAuthImportExport data){
-		ExternalSelectDataPersist.ExternalSelectAuthDataPersist persist = new ExternalSelectDataPersist.ExternalSelectAuthDataPersist();
-		if (data == null) return persist;
-
-		persist.setBody(data.getBody());
-		persist.setUrl(data.getUrl());
-		persist.setMethod(data.getMethod());
-		persist.setPath(data.getPath());
-		persist.setType(data.getType());
-
-		return persist;
-	}
-
-	@Override
-	protected ExternalSelectImportExport dataToImportExportXmlInternal(ExternalSelectDataEntity data, ExternalSelectImportExport xml) {
-		xml.setMultipleSelect(data.getMultipleSelect());
-		if (!this.conventionService.isListNullOrEmpty(data.getSources())){
-			xml.setSources(new ArrayList<>());
-			for (ExternalSelectDataEntity.ExternalSelectSourceEntity externalSelectSourceEntity : data.getSources()) {
-				xml.getSources().add(this.buildAutoCompleteSingleData(externalSelectSourceEntity));
-			}
-		}
-		return xml;
-	}
-
-	@Override
-	protected boolean isMultiValueInternal(ExternalSelectDataEntity data) {
-		return data.getMultipleSelect();
-	}
-
-	private @NotNull ExternalSelectImportExport.ExternalSelectSourceImportExport buildAutoCompleteSingleData(ExternalSelectDataEntity.ExternalSelectSourceEntity data){
-		ExternalSelectImportExport.ExternalSelectSourceImportExport xml = new ExternalSelectImportExport.ExternalSelectSourceImportExport();
-		if (data == null) return xml;
-
-		xml.setUrl(data.getUrl());
-		xml.setOptionsRoot(data.getOptionsRoot());
-		xml.setHasAuth(data.getHasAuth());
-		if(data.getSourceBinding() != null) xml.setSourceBinding(this.buildOption(data.getSourceBinding()));
-		if(data.getAuth() != null) xml.setAuth(this.buildAuthAutoCompleteData(data.getAuth()));
-		xml.setMethod(data.getMethod());
-
-		return xml;
-	}
-
-	private @NotNull ExternalSelectImportExport.ExternalSelectSourceBindingImportExport buildOption(ExternalSelectDataEntity.ExternalSelectSourceBindingEntity data){
-		ExternalSelectImportExport.ExternalSelectSourceBindingImportExport xml = new ExternalSelectImportExport.ExternalSelectSourceBindingImportExport();
-		if (data == null) return xml;
-
-		xml.setLabel(data.getLabel());
-		xml.setValue(data.getValue());
-		xml.setSource(data.getSource());
-
-		return xml;
-	}
-
-	private @NotNull ExternalSelectImportExport.ExternalSelectAuthImportExport buildAuthAutoCompleteData(ExternalSelectDataEntity.ExternalSelectAuthDataEntity data){
-		ExternalSelectImportExport.ExternalSelectAuthImportExport xml = new ExternalSelectImportExport.ExternalSelectAuthImportExport();
-		if (data == null) return xml;
-
-		xml.setBody(data.getBody());
-		xml.setUrl(data.getUrl());
-		xml.setMethod(data.getMethod());
-		xml.setPath(data.getPath());
-		xml.setType(data.getType());
-
-		return xml;
-	}
-}

dmp-backend/core/src/main/java/eu/eudat/service/fielddatahelper/FieldDataHelperServiceProvider.java

@@ -15,19 +15,8 @@ public class FieldDataHelperServiceProvider {
 		BaseFieldDataHelperService<?, ?, ?, ? > item = null;
 		
 		switch (type) {
-			case DATA_REPOSITORIES:
-			case PUB_REPOSITORIES:
-			case JOURNAL_REPOSITORIES:
-			case TAXONOMIES:
-			case LICENSES:
-			case PUBLICATIONS:
-			case REGISTRIES:
-			case SERVICES:
-			case RESEARCHERS:
 			case INTERNAL_DMP_ENTRIES_DATASETS:
-			case INTERNAL_DMP_ENTRIES_DMPS:
+			case INTERNAL_DMP_ENTRIES_DMPS: {
-			case INTERNAL_DMP_ENTRIES_RESEARCHERS:
-			case ORGANIZATIONS: {
 				item = this.applicationContext.getBean(LabelAndMultiplicityFieldDataHelperService.class);
 				break;
 			}
@@ -60,8 +49,8 @@ public class FieldDataHelperServiceProvider {
 				item = this.applicationContext.getBean(SelectFieldDataHelperService.class);
 				break;
 			}
-			case EXTERNAL_SELECT: {
+			case REFERENCE_TYPES:{
-				item = this.applicationContext.getBean(ExternalSelectFieldDataHelperService.class);
+				item = this.applicationContext.getBean(ReferenceTypeFieldDataHelperService.class);
 				break;
 			}
 			default: throw new RuntimeException("unrecognized builder " + type.getValue());

dmp-backend/core/src/main/java/eu/eudat/service/fielddatahelper/ReferenceTypeFieldDataHelperService.java

@@ -0,0 +1,102 @@
+package eu.eudat.service.fielddatahelper;
+
+import eu.eudat.authorization.AuthorizationFlags;
+import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity;
+import eu.eudat.commons.types.descriptiontemplate.importexport.fielddata.ReferenceTypeDataImportExport;
+import eu.eudat.model.builder.descriptiontemplatedefinition.fielddata.ReferenceTypeDataBuilder;
+import eu.eudat.model.descriptiontemplatedefinition.fielddata.ReferenceTypeData;
+import eu.eudat.model.persist.descriptiontemplatedefinition.fielddata.ReferenceTypeDataPersist;
+import gr.cite.tools.data.builder.BuilderFactory;
+import gr.cite.tools.fieldset.FieldSet;
+import gr.cite.tools.validation.Validator;
+import gr.cite.tools.validation.ValidatorFactory;
+import org.springframework.stereotype.Component;
+
+import java.util.EnumSet;
+import java.util.List;
+
+@Component
+public class ReferenceTypeFieldDataHelperService extends BaseFieldDataHelperService<ReferenceTypeData, ReferenceTypeDataPersist, ReferenceTypeDataEntity, ReferenceTypeDataImportExport> {
+	
+	private final BuilderFactory builderFactory;
+
+	private final ValidatorFactory validatorFactory;
+	public ReferenceTypeFieldDataHelperService(BuilderFactory builderFactory, ValidatorFactory validatorFactory) {
+		this.builderFactory = builderFactory;
+		this.validatorFactory = validatorFactory;
+	}
+
+	@Override
+	public ReferenceTypeDataEntity newDataInstanceInternal() {
+		return new ReferenceTypeDataEntity();
+	}
+
+	@Override
+	public ReferenceTypeData newModelInstanceInternal() {
+		return new ReferenceTypeData();
+	}
+
+	@Override
+	public ReferenceTypeDataPersist newPersistModelInstanceInternal() {
+		return new ReferenceTypeDataPersist();
+	}
+
+	@Override
+	protected ReferenceTypeDataImportExport newImportExportInstanceInternal() {
+		return new ReferenceTypeDataImportExport();
+	}
+
+	@Override
+	public Class<?> getDataClass() {
+		return ReferenceTypeDataEntity.class;
+	}
+	@Override
+	public Class<?> getModelClass() {
+		return ReferenceTypeData.class;
+	}
+	@Override
+	public Class<?> getPersistModelClass() {
+		return ReferenceTypeDataPersist.class;
+	}
+
+	@Override
+	public Class<?> getImportExportClass() {
+		return ReferenceTypeDataImportExport.class;
+	}
+	
+	@Override
+	public Validator getPersistModelValidator() {
+		return this.validatorFactory.validator(ReferenceTypeDataPersist.ReferenceTypeDataPersistValidator.class);
+	}
+
+	@Override
+	public List<ReferenceTypeData> buildInternal(FieldSet fieldSet, List<ReferenceTypeDataEntity> data, EnumSet<AuthorizationFlags> authorizationFlags){
+		return this.builderFactory.builder(ReferenceTypeDataBuilder.class).authorize(authorizationFlags).build(fieldSet, data);
+	}
+
+	@Override
+	protected ReferenceTypeDataEntity applyPersistInternal(ReferenceTypeDataPersist persist, ReferenceTypeDataEntity data) {
+		data.setMultipleSelect(persist.getMultipleSelect());
+		data.setReferenceTypeId(persist.getReferenceTypeId());
+		return data;
+	}
+
+	@Override
+	protected ReferenceTypeDataPersist importExportMapDataToPersistInternal(ReferenceTypeDataImportExport data, ReferenceTypeDataPersist persist){
+		persist.setMultipleSelect(data.getMultipleSelect());
+		persist.setReferenceTypeId(data.getReferenceTypeId());
+		return persist;
+	}
+
+	@Override
+	protected ReferenceTypeDataImportExport dataToImportExportXmlInternal(ReferenceTypeDataEntity data, ReferenceTypeDataImportExport xml) {
+		xml.setMultipleSelect(data.getMultipleSelect());
+		xml.setReferenceTypeId(data.getReferenceTypeId());
+		return xml;
+	}
+
+	@Override
+	protected boolean isMultiValueInternal(ReferenceTypeDataEntity data) {
+		return data.getMultipleSelect();
+	}
+}

dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java

@@ -5,38 +5,24 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.authorization.AuthorizationFlags;
 import eu.eudat.commons.JsonHandlingService;
 import eu.eudat.commons.XmlHandlingService;
-import eu.eudat.commons.enums.FieldType;
 import eu.eudat.commons.types.descriptiontemplate.FieldEntity;
 import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity;
-import eu.eudat.commons.types.descriptiontemplate.fielddata.ExternalSelectDataEntity;
-import eu.eudat.commons.types.descriptiontemplate.fielddata.LabelAndMultiplicityDataEntity;
-import eu.eudat.commons.types.descriptiontemplate.fielddata.SelectDataEntity;
 import eu.eudat.commons.types.prefilling.PrefillingEntity;
 import eu.eudat.convention.ConventionService;
 import eu.eudat.data.DescriptionTemplateEntity;
-import eu.eudat.data.ReferenceEntity;
 import eu.eudat.model.*;
 import eu.eudat.model.builder.DescriptionTemplateBuilder;
 import eu.eudat.model.builder.PrefillingBuilder;
-import eu.eudat.model.builder.ReferenceBuilder;
 import eu.eudat.model.descriptionproperties.Field;
 import eu.eudat.model.descriptionproperties.PropertyDefinition;
 import eu.eudat.model.descriptionproperties.PropertyDefinitionFieldSet;
 import eu.eudat.model.descriptionproperties.PropertyDefinitionFieldSetItem;
 import eu.eudat.model.persist.DescriptionProfilingRequest;
 import eu.eudat.model.persist.DescriptionProfilingWithDataRequest;
-import eu.eudat.query.ReferenceQuery;
 import eu.eudat.service.remotefetcher.ExternalUrlConfigProvider;
 import eu.eudat.service.remotefetcher.RemoteFetcherService;
-import eu.eudat.service.remotefetcher.config.AuthenticationConfiguration;
-import eu.eudat.service.remotefetcher.config.DataFieldsUrlConfiguration;
-import eu.eudat.service.remotefetcher.config.DataUrlConfiguration;
-import eu.eudat.service.remotefetcher.config.UrlConfiguration;
 import eu.eudat.service.remotefetcher.config.entities.*;
 import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
-import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
-import eu.eudat.service.remotefetcher.models.ExternalAutocompleteFieldResult;
-import eu.eudat.utilities.helpers.StreamDistinctBy;
 import gr.cite.tools.data.builder.BuilderFactory;
 import gr.cite.tools.data.query.QueryFactory;
 import gr.cite.tools.exception.MyApplicationException;
@@ -47,7 +33,6 @@ import gr.cite.tools.logging.MapLogEntry;
 import gr.cite.tools.validation.ValidatorFactory;
 import jakarta.persistence.EntityManager;
 import jakarta.xml.bind.JAXBException;
-import org.json.JSONObject;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.MessageSource;
@@ -62,8 +47,6 @@ import org.xml.sax.SAXException;
 
 import javax.xml.parsers.ParserConfigurationException;
 import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
@@ -80,11 +63,11 @@ public class PrefillingServiceImpl implements PrefillingService {
     private final QueryFactory queryFactory;
     private final ConventionService conventionService;
     private final MessageSource messageSource;
-    private final RemoteFetcherService remoteFetcherService;
     private final JsonHandlingService jsonHandlingService;
     private final ExternalUrlConfigProvider externalUrlConfigProvider;
     private final XmlHandlingService xmlHandlingService;
     private final ValidatorFactory validatorFactory;
+    private final RemoteFetcherService remoteFetcherService;
     @Autowired
     public PrefillingServiceImpl(
 		    EntityManager entityManager,
@@ -92,21 +75,21 @@ public class PrefillingServiceImpl implements PrefillingService {
 		    QueryFactory queryFactory,
 		    ConventionService conventionService,
 		    MessageSource messageSource,
-		    RemoteFetcherService remoteFetcherService,
 		    JsonHandlingService jsonHandlingService,
 		    ExternalUrlConfigProvider externalUrlConfigProvider,
 		    XmlHandlingService xmlHandlingService,
-            ValidatorFactory validatorFactory) {
+		    ValidatorFactory validatorFactory, 
+            RemoteFetcherService remoteFetcherService) {
         this.entityManager = entityManager;
         this.builderFactory = builderFactory;
         this.queryFactory = queryFactory;
         this.conventionService = conventionService;
         this.messageSource = messageSource;
-	    this.remoteFetcherService = remoteFetcherService;
 	    this.jsonHandlingService = jsonHandlingService;
 	    this.externalUrlConfigProvider = externalUrlConfigProvider;
 	    this.xmlHandlingService = xmlHandlingService;
 	    this.validatorFactory = validatorFactory;
+	    this.remoteFetcherService = remoteFetcherService;
     }
 
     @Override
@@ -118,17 +101,17 @@ public class PrefillingServiceImpl implements PrefillingService {
         List<PrefillingEntity> prefillings = new ArrayList<>();
         List<Map<String, String>> map;
         Map<String, PrefillingConfig> prefillingConfigs = this.externalUrlConfigProvider.getExternalUrls().getPrefillings();
-        for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) {
+//        for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) {//TODO new reference logic
-            map = remoteFetcherService.getExternalGeneric(externalReferenceCriteria, prefillingConfig.getPrefillingSearch());
+//            map = remoteFetcherService.getExternalData(prefillingConfig.getPrefillingSearch().getUrls(), externalReferenceCriteria, prefillingConfig.getPrefillingSearch().getFetchMode());
-            prefillings.addAll(map.stream().map(submap -> PrefillingEntity.build(submap, this.jsonHandlingService)).toList());
+//            prefillings.addAll(map.stream().map(submap -> PrefillingEntity.build(submap, this.jsonHandlingService)).toList());
-            if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) {
+//            if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) {
-                List<Map<String, Object>> mapData = remoteFetcherService.getExternalGenericWithData(externalReferenceCriteria, prefillingConfig.getPrefillingSearch());
+//                List<Map<String, Object>> mapData = remoteFetcherService.getExternalGenericWithData(externalReferenceCriteria, prefillingConfig.getPrefillingSearch());
-                for (int i = 0; i < mapData.size(); i++) {
+//                for (int i = 0; i < mapData.size(); i++) {
-                    prefillings.get(i).setData(mapData.get(i));
+//                    prefillings.get(i).setData(mapData.get(i));
-                }
+//                }
-                prefillings = prefillings.stream().filter(prefilling -> prefilling.getData() != null).collect(Collectors.toList());
+//                prefillings = prefillings.stream().filter(prefilling -> prefilling.getData() != null).collect(Collectors.toList());
-            }
+//            }
-        }
+//        }
         return this.builderFactory.builder(PrefillingBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(BaseFieldSet.build(lookup.getProject(), Prefilling._pid), prefillings);
     }
 
@@ -381,49 +364,49 @@ public class PrefillingServiceImpl implements PrefillingService {
         return parsedValues.isEmpty() ? null : String.join(", ", parsedValues);
     }
 
-    private Object parseComboBoxValues(FieldEntity fieldEntity, List<String> parsedValues)  {
+//    private Object parseComboBoxValues(FieldEntity fieldEntity, List<String> parsedValues)  {
-        List<String> normalizedValues = new ArrayList<>();
+//        List<String> normalizedValues = new ArrayList<>();
-        boolean isMultiSelect;
+//        boolean isMultiSelect;
-        if(fieldEntity.getData().getFieldType().equals(FieldType.EXTERNAL_SELECT)) {
+//        if(fieldEntity.getData().getFieldType().equals(FieldType.EXTERNAL_SELECT)) {
-            ExternalSelectDataEntity autoCompleteData = (ExternalSelectDataEntity)fieldEntity.getData();
+//            ExternalSelectDataEntity autoCompleteData = (ExternalSelectDataEntity)fieldEntity.getData();
-            isMultiSelect = autoCompleteData.getMultipleSelect();
+//            isMultiSelect = autoCompleteData.getMultipleSelect();
-            for (String format : parsedValues) {
+//            for (String format : parsedValues) {
-                List<ExternalAutocompleteFieldResult> result = new ArrayList<>();
+//                List<ExternalAutocompleteFieldResult> result = new ArrayList<>();
-                try {
+//                try {
-                    result = this.getAutocomplete(autoCompleteData, format);
+//                    result = this.getAutocomplete(autoCompleteData, format);
-                }
+//                }
-                catch (Exception e) {
+//                catch (Exception e) {
-                    logger.error(e.getMessage(), e);
+//                    logger.error(e.getMessage(), e);
-                }
+//                }
-                result = result.stream().filter(StreamDistinctBy.distinctByKey(ExternalAutocompleteFieldResult::getId)).collect(Collectors.toList());
+//                result = result.stream().filter(StreamDistinctBy.distinctByKey(ExternalAutocompleteFieldResult::getId)).collect(Collectors.toList());
-                if(!result.isEmpty()){
+//                if(!result.isEmpty()){
-                    List<String> tempValues = new LinkedList<>();
+//                    List<String> tempValues = new LinkedList<>();
-                    for (ExternalAutocompleteFieldResult f : result) {
+//                    for (ExternalAutocompleteFieldResult f : result) {
-                        if (format.equals(f.getId()) || f.getLabel().toUpperCase(Locale.ROOT).contains(format.toUpperCase(Locale.ROOT)))
+//                        if (format.equals(f.getId()) || f.getLabel().toUpperCase(Locale.ROOT).contains(format.toUpperCase(Locale.ROOT)))
-                            tempValues.add(this.jsonHandlingService.toJsonSafe(f));
+//                            tempValues.add(this.jsonHandlingService.toJsonSafe(f));
-                    }
+//                    }
-                    if (isMultiSelect)
+//                    if (isMultiSelect)
-                        normalizedValues.addAll(tempValues);
+//                        normalizedValues.addAll(tempValues);
-                    else if (!tempValues.isEmpty())
+//                    else if (!tempValues.isEmpty())
-                        normalizedValues.add(tempValues.get(0));
+//                        normalizedValues.add(tempValues.get(0));
-                }
+//                }
-            }
+//            }
-            return !normalizedValues.isEmpty() ? (isMultiSelect ? normalizedValues : normalizedValues.getFirst()) : null;
+//            return !normalizedValues.isEmpty() ? (isMultiSelect ? normalizedValues : normalizedValues.getFirst()) : null;
-        } else {
+//        } else {
-            SelectDataEntity selectDataEntity = (SelectDataEntity)fieldEntity.getData();
+//            SelectDataEntity selectDataEntity = (SelectDataEntity)fieldEntity.getData();
-            isMultiSelect = selectDataEntity.getMultipleSelect();
+//            isMultiSelect = selectDataEntity.getMultipleSelect();
-            if (selectDataEntity.getOptions() != null) {
+//            if (selectDataEntity.getOptions() != null) {
-                for (SelectDataEntity.OptionEntity option : selectDataEntity.getOptions()) {
+//                for (SelectDataEntity.OptionEntity option : selectDataEntity.getOptions()) {
-                    if (parsedValues.contains(option.getValue())) {
+//                    if (parsedValues.contains(option.getValue())) {
-                        normalizedValues.add(option.getValue());
+//                        normalizedValues.add(option.getValue());
-                    }
+//                    }
-                }
+//                }
-            }
+//            }
-            List<String> normalizedStringValues = normalizedValues.stream().map(Object::toString).collect(Collectors.toList());
+//            List<String> normalizedStringValues = normalizedValues.stream().map(Object::toString).collect(Collectors.toList());
-
+//
-            return !normalizedValues.isEmpty() ? (isMultiSelect ? String.join(", ", normalizedStringValues) : normalizedValues.getFirst()) : null;
+//            return !normalizedValues.isEmpty() ? (isMultiSelect ? String.join(", ", normalizedStringValues) : normalizedValues.getFirst()) : null;
-        }
+//        }
-    }
+//    }
 
     private static List<Tag> parseTags(String value)  {
         if (value == null || value.isEmpty())
@@ -438,75 +421,75 @@ public class PrefillingServiceImpl implements PrefillingService {
         return parsedTags;
     }
 
-    public List<ExternalAutocompleteFieldResult> getAutocomplete(ExternalSelectDataEntity data, String like) throws URISyntaxException {
+//    public List<ExternalAutocompleteFieldResult> getAutocomplete(ExternalSelectDataEntity data, String like) throws URISyntaxException {
-       
+//       
-        List<ExternalAutocompleteFieldResult> result = new LinkedList<>();
+//        List<ExternalAutocompleteFieldResult> result = new LinkedList<>();
-        ExternalReferenceCriteria urlCriteria = new ExternalReferenceCriteria();
+//        ExternalReferenceCriteria urlCriteria = new ExternalReferenceCriteria();
-        GeneralUrls genericUrls = new GeneralUrls();
+//        GeneralUrls genericUrls = new GeneralUrls();
-        int ordinal = 1;
+//        int ordinal = 1;
-        List<Map<String, String>> rawResults = new ArrayList<>();
+//        List<Map<String, String>> rawResults = new ArrayList<>();
-        genericUrls.setFetchMode(FetchStrategy.FIRST);
+//        genericUrls.setFetchMode(FetchStrategy.FIRST);
-        urlCriteria.setLike(like);
+//        urlCriteria.setLike(like);
-        for (ExternalSelectDataEntity.ExternalSelectSourceEntity singleData : data.getSources()) {
+//        for (ExternalSelectDataEntity.ExternalSelectSourceEntity singleData : data.getSources()) {
-            UrlConfiguration urlConfiguration = new UrlConfiguration();
+//            UrlConfiguration urlConfiguration = new UrlConfiguration();
-            try {
+//            try {
-                URI uri;
+//                URI uri;
-                if (singleData.getUrl().contains("?")) {
+//                if (singleData.getUrl().contains("?")) {
-                    uri = new URI(singleData.getUrl().substring(0, singleData.getUrl().lastIndexOf("?")));
+//                    uri = new URI(singleData.getUrl().substring(0, singleData.getUrl().lastIndexOf("?")));
-                } else {
+//                } else {
-                    uri = new URI(singleData.getUrl());
+//                    uri = new URI(singleData.getUrl());
-                }
+//                }
-                String source = singleData.getSourceBinding().getSource();
+//                String source = singleData.getSourceBinding().getSource();
-                source = source != null && !source.isEmpty() ? source : uri.getHost();
+//                source = source != null && !source.isEmpty() ? source : uri.getHost();
-//                String uriString = singleData.getSourceBinding().getUrl();
+////                String uriString = singleData.getSourceBinding().getUrl();
-//                uriString = uriString != null && !uriString.isEmpty() ? uriString : "uri";
+////                uriString = uriString != null && !uriString.isEmpty() ? uriString : "uri";
-                String uriString = "uri";
+//                String uriString = "uri";
-                String parsedUrl = singleData.getUrl();
+//                String parsedUrl = singleData.getUrl();
-                parsedUrl = parsedUrl.replace("%20", " ");
+//                parsedUrl = parsedUrl.replace("%20", " ");
-                parsedUrl = parsedUrl.replace("%22", "\"");
+//                parsedUrl = parsedUrl.replace("%22", "\"");
-                while (parsedUrl.contains("&amp;")) {
+//                while (parsedUrl.contains("&amp;")) {
-                    parsedUrl = parsedUrl.replace("&amp;", "&");
+//                    parsedUrl = parsedUrl.replace("&amp;", "&");
-                }
+//                }
-                urlConfiguration.setUrl(parsedUrl);
+//                urlConfiguration.setUrl(parsedUrl);
-                urlConfiguration.setOrdinal(ordinal);
+//                urlConfiguration.setOrdinal(ordinal);
-                urlConfiguration.setType("External");
+//                urlConfiguration.setType("External");
-                urlConfiguration.setContentType(MediaType.APPLICATION_JSON_VALUE);
+//                urlConfiguration.setContentType(MediaType.APPLICATION_JSON_VALUE);
-                urlConfiguration.setFirstpage("1");
+//                urlConfiguration.setFirstpage("1");
-                urlConfiguration.setRequestType(singleData.getMethod() != null ? singleData.getMethod() : "GET");
+//                urlConfiguration.setRequestType(singleData.getMethod() != null ? singleData.getMethod() : "GET");
-                DataUrlConfiguration dataUrlConfiguration = new DataUrlConfiguration();
+//                DataUrlConfiguration dataUrlConfiguration = new DataUrlConfiguration();
-                dataUrlConfiguration.setPath(singleData.getOptionsRoot());
+//                dataUrlConfiguration.setPath(singleData.getOptionsRoot());
-                DataFieldsUrlConfiguration fieldsUrlConfiguration = new DataFieldsUrlConfiguration();
+//                DataFieldsUrlConfiguration fieldsUrlConfiguration = new DataFieldsUrlConfiguration();
-                fieldsUrlConfiguration.setId(singleData.getSourceBinding().getValue());
+//                fieldsUrlConfiguration.setId(singleData.getSourceBinding().getValue());
-                fieldsUrlConfiguration.setName(singleData.getSourceBinding().getLabel());
+//                fieldsUrlConfiguration.setName(singleData.getSourceBinding().getLabel());
-                fieldsUrlConfiguration.setSource(singleData.getSourceBinding().getSource().isEmpty()? null : singleData.getSourceBinding().getSource());
+//                fieldsUrlConfiguration.setSource(singleData.getSourceBinding().getSource().isEmpty()? null : singleData.getSourceBinding().getSource());
-                fieldsUrlConfiguration.setUri(uriString);
+//                fieldsUrlConfiguration.setUri(uriString);
-                dataUrlConfiguration.setFieldsUrlConfiguration(fieldsUrlConfiguration);
+//                dataUrlConfiguration.setFieldsUrlConfiguration(fieldsUrlConfiguration);
-                urlConfiguration.setKey(source);
+//                urlConfiguration.setKey(source);
-                urlConfiguration.setLabel(source);
+//                urlConfiguration.setLabel(source);
-                urlConfiguration.setData(dataUrlConfiguration);
+//                urlConfiguration.setData(dataUrlConfiguration);
-                if (singleData.getHasAuth()) {
+//                if (singleData.getHasAuth()) {
-                    AuthenticationConfiguration authenticationConfiguration = new AuthenticationConfiguration();
+//                    AuthenticationConfiguration authenticationConfiguration = new AuthenticationConfiguration();
-                    authenticationConfiguration.setAuthUrl(singleData.getAuth().getUrl());
+//                    authenticationConfiguration.setAuthUrl(singleData.getAuth().getUrl());
-                    authenticationConfiguration.setAuthMethod(singleData.getAuth().getMethod());
+//                    authenticationConfiguration.setAuthMethod(singleData.getAuth().getMethod());
-                    authenticationConfiguration.setAuthTokenPath(singleData.getAuth().getPath());
+//                    authenticationConfiguration.setAuthTokenPath(singleData.getAuth().getPath());
-                    authenticationConfiguration.setAuthRequestBody(singleData.getAuth().getBody());
+//                    authenticationConfiguration.setAuthRequestBody(singleData.getAuth().getBody());
-                    authenticationConfiguration.setType(singleData.getAuth().getType());
+//                    authenticationConfiguration.setType(singleData.getAuth().getType());
-                    urlConfiguration.setAuth(authenticationConfiguration);
+//                    urlConfiguration.setAuth(authenticationConfiguration);
-                }
+//                }
-                genericUrls.getUrls().add(urlConfiguration);
+//                genericUrls.getUrls().add(urlConfiguration);
-                List<Map<String, String>> singleResults =  this.remoteFetcherService.getExternalGeneric(urlCriteria, genericUrls);
+//                List<Map<String, String>> singleResults =  this.remoteFetcherService.getExternalGeneric(urlCriteria, genericUrls);
-                if (!singleResults.isEmpty() && !singleResults.get(0).containsKey("source") && !singleData.getSourceBinding().getSource().isEmpty()) {
+//                if (!singleResults.isEmpty() && !singleResults.get(0).containsKey("source") && !singleData.getSourceBinding().getSource().isEmpty()) {
-                    singleResults.forEach(singleResult -> singleResult.put("source", singleData.getSourceBinding().getSource()));
+//                    singleResults.forEach(singleResult -> singleResult.put("source", singleData.getSourceBinding().getSource()));
-                }
+//                }
-                rawResults.addAll(singleResults);
+//                rawResults.addAll(singleResults);
-                genericUrls.getUrls().clear();
+//                genericUrls.getUrls().clear();
-            } catch (URISyntaxException e) {
+//            } catch (URISyntaxException e) {
-                logger.error(e.getMessage(), e);
+//                logger.error(e.getMessage(), e);
-            }
+//            }
-        }
+//        }
-        rawResults.forEach(item -> result.add(new ExternalAutocompleteFieldResult(parseItem(item.get("pid")), parseItem(item.get("name")), parseItem(item.get("source")), parseItem(item.get("uri")))));
+//        rawResults.forEach(item -> result.add(new ExternalAutocompleteFieldResult(parseItem(item.get("pid")), parseItem(item.get("name")), parseItem(item.get("source")), parseItem(item.get("uri")))));
-        return result;
+//        return result;
-    }
+//    }
 
     private static String parseItem(Object item) {
         if (item instanceof String) {

dmp-backend/core/src/main/java/eu/eudat/service/publicapi/PublicApiConfiguration.java

@@ -0,0 +1,10 @@
+package eu.eudat.service.publicapi;
+
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@EnableConfigurationProperties(PublicApiProperties.class)
+public class PublicApiConfiguration {
+
+}

dmp-backend/core/src/main/java/eu/eudat/service/publicapi/PublicApiProperties.java

@@ -0,0 +1,93 @@
+package eu.eudat.service.publicapi;
+
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import java.util.UUID;
+
+@ConfigurationProperties(prefix = "public-api")
+public class PublicApiProperties {
+	private ReferenceTypeMapConfig referenceTypeMap;
+	public ReferenceTypeMapConfig getReferenceTypeMap() {
+		return referenceTypeMap;
+	}
+
+	public void setReferenceTypeMap(ReferenceTypeMapConfig referenceTypeMap) {
+		this.referenceTypeMap = referenceTypeMap;
+	}
+
+	public static class ReferenceTypeMapConfig{
+		private UUID funderTypeId;
+		private UUID grantTypeId;
+		private UUID registryTypeId;
+		private UUID serviceTypeId;
+		private UUID dataRepositoryTypeId;
+		private UUID datasetTypeId;
+		private UUID researcherTypeId;
+		private UUID organizationTypeId;
+		public UUID getFunderTypeId() {
+			return funderTypeId;
+		}
+
+		public void setFunderTypeId(UUID funderTypeId) {
+			this.funderTypeId = funderTypeId;
+		}
+
+		public UUID getGrantTypeId() {
+			return grantTypeId;
+		}
+
+		public void setGrantTypeId(UUID grantTypeId) {
+			this.grantTypeId = grantTypeId;
+		}
+
+		public UUID getRegistryTypeId() {
+			return registryTypeId;
+		}
+
+		public void setRegistryTypeId(UUID registryTypeId) {
+			this.registryTypeId = registryTypeId;
+		}
+
+		public UUID getServiceTypeId() {
+			return serviceTypeId;
+		}
+
+		public void setServiceTypeId(UUID serviceTypeId) {
+			this.serviceTypeId = serviceTypeId;
+		}
+
+		public UUID getDataRepositoryTypeId() {
+			return dataRepositoryTypeId;
+		}
+
+		public void setDataRepositoryTypeId(UUID dataRepositoryTypeId) {
+			this.dataRepositoryTypeId = dataRepositoryTypeId;
+		}
+
+		public UUID getDatasetTypeId() {
+			return datasetTypeId;
+		}
+
+		public void setDatasetTypeId(UUID datasetTypeId) {
+			this.datasetTypeId = datasetTypeId;
+		}
+
+		public UUID getResearcherTypeId() {
+			return researcherTypeId;
+		}
+
+		public void setResearcherTypeId(UUID researcherTypeId) {
+			this.researcherTypeId = researcherTypeId;
+		}
+
+		public UUID getOrganizationTypeId() {
+			return organizationTypeId;
+		}
+
+		public void setOrganizationTypeId(UUID organizationTypeId) {
+			this.organizationTypeId = organizationTypeId;
+		}
+	}
+}
+

dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java

@@ -2,10 +2,12 @@ package eu.eudat.service.reference;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import eu.eudat.commons.exceptions.HugeResultSetException;
+import eu.eudat.data.ReferenceTypeEntity;
 import eu.eudat.model.Reference;
 import eu.eudat.model.persist.ReferencePersist;
-import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
 import eu.eudat.query.lookup.ReferenceSearchLookup;
+import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
+import eu.eudat.service.remotefetcher.models.ExternalDataResult;
 import gr.cite.tools.exception.MyApplicationException;
 import gr.cite.tools.exception.MyForbiddenException;
 import gr.cite.tools.exception.MyNotFoundException;
@@ -24,7 +26,5 @@ public interface ReferenceService {
 
 	void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException;
 
-	List<Reference> searchReference(ReferenceSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException;
+	List<Reference> searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException;
-
-	List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException;
 }

dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java

@@ -1,18 +1,12 @@
 package eu.eudat.service.reference;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.jayway.jsonpath.DocumentContext;
-import com.jayway.jsonpath.JsonPath;
 import eu.eudat.authorization.AuthorizationFlags;
 import eu.eudat.authorization.Permission;
 import eu.eudat.commons.XmlHandlingService;
 import eu.eudat.commons.enums.IsActive;
+import eu.eudat.commons.enums.ReferenceFieldDataType;
 import eu.eudat.commons.enums.ReferenceSourceType;
-import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
-import eu.eudat.commons.enums.ReferenceTypeSourceType;
-import eu.eudat.commons.exceptions.HugeResultSetException;
 import eu.eudat.commons.types.reference.DefinitionEntity;
 import eu.eudat.commons.types.reference.FieldEntity;
 import eu.eudat.commons.types.referencetype.*;
@@ -20,20 +14,18 @@ import eu.eudat.convention.ConventionService;
 import eu.eudat.data.ReferenceEntity;
 import eu.eudat.data.ReferenceTypeEntity;
 import eu.eudat.model.Reference;
+import eu.eudat.model.ReferenceType;
 import eu.eudat.model.builder.ReferenceBuilder;
-import eu.eudat.model.builder.referencesearch.ReferenceSearchBuilder;
 import eu.eudat.model.deleter.ReferenceDeleter;
 import eu.eudat.model.persist.ReferencePersist;
 import eu.eudat.model.persist.referencedefinition.DefinitionPersist;
 import eu.eudat.model.persist.referencedefinition.FieldPersist;
 import eu.eudat.query.ReferenceQuery;
-import eu.eudat.query.ReferenceTypeQuery;
-import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
 import eu.eudat.query.lookup.ReferenceSearchLookup;
 import eu.eudat.service.remotefetcher.RemoteFetcherService;
+import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
 import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
-import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
+import eu.eudat.service.remotefetcher.models.ExternalDataResult;
-import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
 import gr.cite.commons.web.authz.service.AuthorizationService;
 import gr.cite.tools.data.builder.BuilderFactory;
 import gr.cite.tools.data.deleter.DeleterFactory;
@@ -48,34 +40,22 @@ import gr.cite.tools.logging.LoggerService;
 import gr.cite.tools.logging.MapLogEntry;
 import jakarta.persistence.EntityManager;
 import jakarta.xml.bind.JAXBException;
-import net.minidev.json.JSONArray;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.MessageSource;
 import org.springframework.context.i18n.LocaleContextHolder;
-import org.springframework.core.ParameterizedTypeReference;
-import org.springframework.http.HttpMethod;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
-import org.springframework.http.client.reactive.ReactorClientHttpConnector;
-import org.springframework.http.codec.json.Jackson2JsonDecoder;
 import org.springframework.stereotype.Service;
-import org.springframework.web.reactive.function.client.WebClient;
-import reactor.netty.http.client.HttpClient;
 
 import javax.management.InvalidApplicationException;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.TransformerException;
 import java.time.Instant;
 import java.util.*;
-import java.util.stream.Collectors;
 
 @Service
 public class ReferenceServiceImpl implements ReferenceService {
 
     private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(ReferenceServiceImpl.class));
-    private final RemoteFetcherService remoteFetcherService;
     private final EntityManager entityManager;
     private final AuthorizationService authorizationService;
     private final DeleterFactory deleterFactory;
@@ -85,9 +65,8 @@ public class ReferenceServiceImpl implements ReferenceService {
     private final QueryFactory queryFactory;
     private final XmlHandlingService xmlHandlingService;
 
-    private final WebClient client;
+    public final RemoteFetcherService remoteFetcherService;
-
+    public ReferenceServiceImpl(
-    public ReferenceServiceImpl(RemoteFetcherService remoteFetcherService,
 		    EntityManager entityManager,
 		    AuthorizationService authorizationService,
 		    DeleterFactory deleterFactory,
@@ -95,8 +74,7 @@ public class ReferenceServiceImpl implements ReferenceService {
 		    ConventionService conventionService,
 		    MessageSource messageSource,
 		    QueryFactory queryFactory,
-                                XmlHandlingService xmlHandlingService) {
+		    XmlHandlingService xmlHandlingService, RemoteFetcherService remoteFetcherService) {
-        this.remoteFetcherService = remoteFetcherService;
         this.entityManager = entityManager;
         this.authorizationService = authorizationService;
         this.deleterFactory = deleterFactory;
@@ -105,11 +83,7 @@ public class ReferenceServiceImpl implements ReferenceService {
         this.messageSource = messageSource;
         this.queryFactory = queryFactory;
         this.xmlHandlingService = xmlHandlingService;
-        this.client = WebClient.builder().codecs(clientCodecConfigurer -> {
+	    this.remoteFetcherService = remoteFetcherService;
-                    clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
-                    clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
-                }
-        ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
     }
 
     @Override
@@ -133,7 +107,7 @@ public class ReferenceServiceImpl implements ReferenceService {
         }
 
         data.setLabel(model.getLabel());
-        data.setType(model.getType());
+        data.setTypeId(model.getTypeId());
         data.setDescription(model.getDescription());
         data.setDefinition(this.xmlHandlingService.toXmlSafe(this.buildDefinitionEntity(model.getDefinition())));
         data.setUpdatedAt(Instant.now());
@@ -183,37 +157,39 @@ public class ReferenceServiceImpl implements ReferenceService {
         this.deleterFactory.deleter(ReferenceDeleter.class).deleteAndSaveByIds(List.of(id));
     }
 
-//    public List<FetcherReference> searchReference(ReferenceType externalType, String query, String type) throws HugeResultSet, MyNotFoundException, InvalidApplicationException {
-//        ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
-//
-//        List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().get(externalType, externalUrlCriteria, type);
-//
-//        List<FetcherReference> list = this.fetchFromDb(externalType, query, type, remoteRepos);
-//
-//        list.addAll(remoteRepos.stream().map(FetcherReference::fromRemoteModel).toList());
-//        list = list.stream().filter(x -> x.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
-//        list.sort(Comparator.comparing(FetcherReference::getName));
-//        return list;
-//    }
-
     @Override
-    public List<Reference> searchReference(ReferenceSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException {
+    public List<Reference> searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException {
         int initialOffset = 0;
         if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
             initialOffset =  lookup.getPage().getOffset();
             lookup.getPage().setOffset(0);
         }
 
+        ReferenceTypeEntity data = this.entityManager.find(ReferenceTypeEntity.class, lookup.getTypeId());
+        if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{lookup.getTypeId(), ReferenceType.class.getSimpleName()}, LocaleContextHolder.getLocale()));
+            
         ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike());
 
-        List<Map<String, String>> remoteRepos = remoteFetcherService.getReferences(lookup.getType(), externalReferenceCriteria, lookup.getKey());
+        ExternalDataResult remoteRepos = this.getReferenceData(data, externalReferenceCriteria, lookup.getKey());
 
-        List<Reference> externalModels = this.builderFactory.builder(ReferenceSearchBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), remoteRepos);
+        List<Reference> externalModels = new ArrayList<>();
-        List<Reference> models = this.fetchFromDb(lookup);
+        if (remoteRepos != null && this.conventionService.isListNullOrEmpty(remoteRepos.getResults())){
+            List<ReferenceEntity> referenceEntities = new ArrayList<>();
+            for (Map<String, String> result : remoteRepos.getResults()){
+                if (result == null || result.isEmpty()) continue;;
+                ReferenceEntity referenceEntity = buildReferenceEntityFromExternalData(result, data, remoteRepos);
+                referenceEntities.add(referenceEntity);
+            }
+            externalModels = this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), referenceEntities);
+        }
         
-        models.addAll(externalModels);
+        List<Reference> models = this.fetchReferenceFromDb(lookup);
 
-        if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).collect(Collectors.toList()); }
+        for (Reference externalReference : externalModels){
+            if (models.stream().noneMatch(x-> x.getReference() != null && x.getSource() != null &&x.getReference().equals(externalReference.getReference()) && x.getSource().equals(externalReference.getSource()))) models.add(externalReference);
+        }
+
+        if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).toList(); }
         models.sort(Comparator.comparing(Reference::getLabel));
 
         if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
@@ -223,340 +199,298 @@ public class ReferenceServiceImpl implements ReferenceService {
         return models;
     }
 
-    private List<Reference> fetchFromDb(ReferenceSearchLookup lookup){
+    @NotNull
-        ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal);
+    private ReferenceEntity buildReferenceEntityFromExternalData(Map<String, String> result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) {
+        ReferenceEntity referenceEntity = new ReferenceEntity(); //TODO new reference logic
+        referenceEntity.setTypeId(data.getId());
+        referenceEntity.setIsActive(IsActive.Active);
+        referenceEntity.setReference(result.getOrDefault("key", null) + ":" + remoteRepos.getResults().getFirst().getOrDefault("pid", null));
+        referenceEntity.setSource(result.getOrDefault("tag", null));
+        referenceEntity.setAbbreviation(result.getOrDefault("abbreviation", null));
+        referenceEntity.setDescription(result.getOrDefault("description", null));
+        referenceEntity.setLabel(result.getOrDefault("name", null));
+        referenceEntity.setSourceType(ReferenceSourceType.External);
+        DefinitionEntity definitionEntity = new DefinitionEntity();
+        definitionEntity.setFields(new ArrayList<>());
+        for (Map.Entry<String, String> resultValue : result.entrySet()){
+            FieldEntity fieldEntity = new FieldEntity();
+            fieldEntity.setCode(resultValue.getKey());
+            fieldEntity.setValue(resultValue.getValue());
+            fieldEntity.setDataType(ReferenceFieldDataType.Text);
+            definitionEntity.getFields().add(fieldEntity);
+        }
+        referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
+        return referenceEntity;
+    }
+
+    private List<Reference> fetchReferenceFromDb(ReferenceSearchLookup lookup){
+        ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal).typeIds(lookup.getTypeId());
         List<ReferenceEntity> data = query.collectAs(lookup.getProject());
         return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data);
     }
 
-//    private List<FetcherReference> fetchFromDb(ReferenceType externalType, String query, String type, List<Map<String, String>> remoteRepos) throws InvalidApplicationException {
+    private ExternalDataResult getReferenceData(ReferenceTypeEntity referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key)  {
-//        List<FetcherReference> list = new LinkedList<>();
+        ReferenceTypeDefinitionEntity referenceTypeDefinition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition());
-//        switch (externalType) {
+        if (referenceTypeDefinition == null || this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) return new ExternalDataResult();
-//            case DataRepositories:
-//            case PubRepositories:
-//            case Journals: {
-//                DataRepositoryCriteria criteria = new DataRepositoryCriteria();
-//                if (!query.isEmpty()) criteria.setLike(query);
-//                criteria.setCreationUserId(this.userScope.getUserId());
-//                if (type.equals("")) {
-//                    List<DataRepository> dataRepositoryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao().getWithCriteria(criteria)).toList();
-//                    list = dataRepositoryList.stream().map(item -> new FetcherReference().fromDataRepository(item)).collect(Collectors.toList());
-//                }
-//            }
-//            case Registries: {
-//                RegistryCriteria criteria = new RegistryCriteria();
-//                if (!query.isEmpty()) criteria.setLike(query);
-//                criteria.setCreationUserId(this.userScope.getUserId());
-//                if (type.equals("")) {
-//                    List<Registry> registryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao().getWithCriteria(criteria)).toList();
-//                    list = registryList.stream().map(item -> new FetcherReference().fromRegistry(item)).collect(Collectors.toList());
-//                }
-//            }
-//            case Services:
-//            {
-//                ServiceCriteria criteria = new ServiceCriteria();
-//
-//                if (!query.isEmpty()) criteria.setLike(query);
-//                criteria.setCreationUserId(this.userScope.getUserId());
-//
-//                if (type.equals("")) {
-//                    List<Service> serviceList = (this.apiContext.getOperationsContext().getDatabaseRepository().getServiceDao().getWithCriteria(criteria)).toList();
-//                   list = serviceList.stream().map(item -> new FetcherReference().fromService(item)).collect(Collectors.toList());
-//                }
-//            }
-//            case Datasets:{
-//
-//                ExternalDatasetCriteria criteria = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ExternalDatasetCriteriaBuilder.class).like(query).build();
-//
-//                criteria.setCreationUserId(this.userScope.getUserId());
-//                QueryableList<ExternalDataset> items = apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao().getWithCriteria(criteria);
-//
-//                list = items.select(item -> new FetcherReference().fromDataset(item));
-//            }
-//            case Taxonomies:
-//            case Publications:
-//            case Licenses:
-//                break;
-//        }
-//
-//        return list;
-//    }
         
-    @Override
+        ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).toList(), externalReferenceCriteria,  key, null);
-    public List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException {
+        for (Map<String, String> result: results.getResults()) {
-
+            result.put("referenceType", referenceType.getName());
-        ReferenceTypeQuery query = this.queryFactory.query(ReferenceTypeQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(lookup.getReferenceTypeId());
-        List<ReferenceTypeEntity> datas = query.collectAs(lookup.getProject());
-        if (datas.size() != 1 ){
-            return null;
         }
-        ReferenceTypeDefinitionEntity definition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, datas.get(0).getDefinition());
-        List<Map<String, String>> remoteRepos = this.getAll(definition.getSources(), lookup);
-
-        return null;
-    }
-
-    private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
-        List<Map<String, String>> results = new LinkedList<>();
-
-        if (sources == null || sources.isEmpty()) {
         return results;
     }
 
-        sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
+//    private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
+//        List<Map<String, String>> results = new LinkedList<>();
+//
+//        if (sources == null || sources.isEmpty()) {
+//            return results;
+//        }
+//
+//        sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
+//
+//        List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
+//        apiSources.forEach(source -> {
+//                try {
+//                    String auth = null;
+//                    if (source.getAuth()!= null) {
+//                        //auth = this.getAuthentication(source.getAuth());
+//                    }
+//                    results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
+//                } catch (Exception e) {
+//                    logger.error(e.getLocalizedMessage(), e);
+//                }
+//        });
+//        List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
+//        staticSources.forEach(source -> {
+//            Map<String, String> map = new HashMap<>();
+//            source.getOptions().forEach(option -> {
+//                map.put(option.getCode(), option.getValue());
+//                map.put("tag", source.getLabel());
+//                map.put("key", source.getKey());
+//            });
+//            results.add(map);
+//        });
+//
+//        return results;
+//    }
 
-        List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
+//    private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) {
-        apiSources.forEach(source -> {
+//        HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name());
-                try {
+//        Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl())
-                    String auth = null;
+//                .contentType(MediaType.APPLICATION_JSON)
-                    if (source.getAuth()!= null) {
+//                .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
-                        //auth = this.getAuthentication(source.getAuth());
+//                .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
-                    }
+//                })).block();
-                    results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
+//
-                } catch (Exception e) {
+//
-                    logger.error(e.getLocalizedMessage(), e);
+//
-                }
+//        return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
-        });
+//    }
-        List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
-        staticSources.forEach(source -> {
-            Map<String, String> map = new HashMap<>();
-            source.getOptions().forEach(option -> {
-                map.put(option.getCode(), option.getValue());
-                map.put("tag", source.getLabel());
-                map.put("key", source.getKey());
-            });
-            results.add(map);
-        });
 
-        return results;
+//    private String parseBodyString(String bodyString) {
-    }
+//        String finalBodyString = bodyString;
+//        if (bodyString.contains("{env:")) {
+//            int index = bodyString.indexOf("{env: ");
+//            while (index >= 0) {
+//                int endIndex = bodyString.indexOf("}", index + 6);
+//                String envName = bodyString.substring(index + 6, endIndex);
+//                finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
+//                index = bodyString.indexOf("{env: ", index + 6);
+//            }
+//        }
+//        return finalBodyString;
+//    }
 
-    private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) {
+//    private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
-        HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name());
+//        Set<Integer> pages = new HashSet<>();
-        Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl())
+//
-                .contentType(MediaType.APPLICATION_JSON)
+//        String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
-                .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
+//        String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
-                .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
+//
-                })).block();
+//        ExternalDataResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
+//        if(externalRefernceResult != null) {
+//            if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
+//                externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
+//                        .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
+//                        .collect(Collectors.toList()));
+//            }
+//            if (fetchStrategy == FetchStrategy.FIRST)
+//                return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
+//
+//            //Long maxResults = configLoader.getExternalUrls().getMaxresults();
+//            Long maxResults = Long.valueOf(1000);
+//            if (externalRefernceResult.getResults().size() > maxResults)
+//                throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getResults().size() + " results... Please submit a more detailed search query");
+//
+//            Optional<ExternalDataResult> optionalResults = pages.parallelStream()
+//                    .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
+//                    .filter(Objects::nonNull)
+//                    .reduce((result1, result2) -> {
+//                        result1.getResults().addAll(result2.getResults());
+//                        return result1;
+//                    });
+//            ExternalDataResult remainingExternalDataResult = optionalResults.orElseGet(ExternalDataResult::new);
+//            remainingExternalDataResult.getResults().addAll(externalRefernceResult.getResults());
+//
+//            return remainingExternalDataResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
+//        }
+//        else {
+//            return new LinkedList<>();
+//        }
+//    }
+
+//    private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
+//        String completedPath = urlPath;
+//
+//        if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
+//            if (lookup.getLike() != null) {
+//                completedPath = completedPath.replace("{like}", lookup.getLike());
+//            } else {
+//                completedPath = completedPath.replace("{like}", "*");
+//            }
+//        }
+//
+//        if (urlPath.contains("{like}")){
+//            if (lookup.getLike() != null) {
+//                completedPath = completedPath.replace("{like}", lookup.getLike());
+//            } else {
+//                completedPath = completedPath.replace("{like}", "");
+//            }
+//        }
+//
+//        if (urlPath.contains("{page}")) {
+//            if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
+//                completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
+//            } else if (firstPage != null) {
+//                completedPath = completedPath.replace("{page}", firstPage);
+//            } else {
+//                completedPath = completedPath.replace("{page}", "1");
+//            }
+//        }
+//
+//        if (urlPath.contains("{pageSize}")){
+//            if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
+//                completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
+//            } else {
+//                completedPath = completedPath.replace("{pageSize}", "100");
+//            }
+//        }
+//
+//
+//        return completedPath;
+//    }
+
+//    protected ExternalDataResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
+//
+//        try {
+//            ResponseEntity<String> response;
+//            JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
+//
+//            response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> {
+//                if (contentType != null && !contentType.isEmpty()) {
+//                    httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
+//                    httpHeaders.setContentType(MediaType.valueOf(contentType));
+//                }
+//                if (auth != null) {
+//                    httpHeaders.set("Authorization", auth);
+//                }
+//            }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
+//            if (response.getStatusCode() == HttpStatus.OK) { // success
+//                ExternalDataResult externalDataResult = new ExternalDataResult();
+//                if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
+//                    DocumentContext jsonContext = JsonPath.parse(response.getBody());
+//                    externalDataResult = this.getFromJson(jsonContext, resultsEntity);
+//                }
+//
+//                return externalDataResult;
+//            }
+//        } catch (Exception exception) {
+//            logger.error(exception.getMessage(), exception);
+//        }
+//
+//        return null;
+//    }
 
 
-
+//    public static ExternalDataResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
-        return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
+//        return new ExternalDataResult(parseData(jsonContext, resultsEntity));
-    }
+//    }
-
-    private String parseBodyString(String bodyString) {
-        String finalBodyString = bodyString;
-        if (bodyString.contains("{env:")) {
-            int index = bodyString.indexOf("{env: ");
-            while (index >= 0) {
-                int endIndex = bodyString.indexOf("}", index + 6);
-                String envName = bodyString.substring(index + 6, endIndex);
-                finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
-                index = bodyString.indexOf("{env: ", index + 6);
-            }
-        }
-        return finalBodyString;
-    }
-
-    private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
-        Set<Integer> pages = new HashSet<>();
-
-        String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
-        String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
-
-        ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
-        if(externalRefernceResult != null) {
-            if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
-                externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
-                        .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
-                        .collect(Collectors.toList()));
-            }
-            if (fetchStrategy == FetchStrategy.FIRST)
-                return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
-
-            if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set
-                for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++)
-                    pages.add(i);
-
-            //Long maxResults = configLoader.getExternalUrls().getMaxresults();
-            Long maxResults = Long.valueOf(1000);
-            if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults))
-                throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query");
-
-            Optional<ExternalRefernceResult> optionalResults = pages.parallelStream()
-                    .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
-                    .filter(Objects::nonNull)
-                    .reduce((result1, result2) -> {
-                        result1.getResults().addAll(result2.getResults());
-                        return result1;
-                    });
-            ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new);
-            remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults());
-
-            return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
-        }
-        else {
-            return new LinkedList<>();
-        }
-    }
-
-    private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
-        String completedPath = urlPath;
-
-        if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
-            if (lookup.getLike() != null) {
-                completedPath = completedPath.replace("{like}", lookup.getLike());
-            } else {
-                completedPath = completedPath.replace("{like}", "*");
-            }
-        }
-
-        if (urlPath.contains("{like}")){
-            if (lookup.getLike() != null) {
-                completedPath = completedPath.replace("{like}", lookup.getLike());
-            } else {
-                completedPath = completedPath.replace("{like}", "");
-            }
-        }
-
-        if (urlPath.contains("{page}")) {
-            if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
-                completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
-            } else if (firstPage != null) {
-                completedPath = completedPath.replace("{page}", firstPage);
-            } else {
-                completedPath = completedPath.replace("{page}", "1");
-            }
-        }
-
-        if (urlPath.contains("{pageSize}")){
-            if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
-                completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
-            } else {
-                completedPath = completedPath.replace("{pageSize}", "100");
-            }
-        }
 
 
-        return completedPath;
+//    private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
-    }
+//        List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath());
-
+//        List<Map<String, String>> parsedData = new ArrayList<>();
-    protected ExternalRefernceResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
+//
-
+//        for (Map<String, String> stringObjectMap: rawData){
-        try {
+//            Map<String, String> map = new HashMap<>();
-            ResponseEntity<String> response;
+//            for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){
-            JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
+//                String pathValue = field.getResponsePath();
-
+//                if (!pathValue.contains(".")){
-            response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> {
+//                    if (stringObjectMap.containsKey(pathValue)) {
-                if (contentType != null && !contentType.isEmpty()) {
+//                        //map.put(field.getCode(), stringObjectMap.get(pathValue));
-                    httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
+//                        map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
-                    httpHeaders.setContentType(MediaType.valueOf(contentType));
+//                    }
-                }
+//                }else {
-                if (auth != null) {
+//                    if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
-                    httpHeaders.set("Authorization", auth);
+//                        String value = null;
-                }
+//                        Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
-            }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
+//                        if (fieldObj != null){
-            if (response.getStatusCode() == HttpStatus.OK) { // success
+//                            if (fieldObj instanceof Map){
-                ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult();
+//                                Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
-                if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
+//                                if(o instanceof String){
-                    DocumentContext jsonContext = JsonPath.parse(response.getBody());
+//                                    value = (String)o;
-                    externalRefernceResult = this.getFromJson(jsonContext, resultsEntity);
+//                                }
-                }
+//                                else if(o instanceof Integer){
-
+//                                    value = String.valueOf(o);
-
+//                                }
-                if (externalRefernceResult.getPagination().size() == 0) {
+//                            } else if (fieldObj instanceof List) {
-                    externalRefernceResult.getPagination().put("pages", 1);
+//                                Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
-                    externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size());
+//                                if(o instanceof String){
-                }
+//                                    value = (String)o;
-                return externalRefernceResult;
+//                                }
-            }
+//                                else if(o instanceof Integer){
-        } catch (Exception exception) {
+//                                    value = String.valueOf(o);
-            logger.error(exception.getMessage(), exception);
+//                                }
-        }
+//                            }
-
+//                        }
-        return null;
+//                        if (value != null){
-    }
+//                            map.put(field.getCode(), value);
-
+//                        }
-
+//                    }
-    public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
+//                }
-        return new ExternalRefernceResult(parseData(jsonContext, resultsEntity),
+//            }
-                new HashMap<>(1, 1));
+//            parsedData.add(map);
-    }
+//        }
-
+//        return parsedData;
-
+//    }
-    private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
+//
-        List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath());
+//
-        List<Map<String, String>> parsedData = new ArrayList<>();
+//    private static String normalizeValue(Object value) {
-
+//        if (value instanceof JSONArray) {
-        for (Map<String, String> stringObjectMap: rawData){
+//            JSONArray jarr = (JSONArray) value;
-            Map<String, String> map = new HashMap<>();
+//            if (jarr.get(0) instanceof String) {
-            for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){
+//                return jarr.get(0).toString();
-                String pathValue = field.getResponsePath();
+//            } else {
-                if (!pathValue.contains(".")){
+//                for (Object o : jarr) {
-                    if (stringObjectMap.containsKey(pathValue)) {
+//                    if ((o instanceof Map) && ((Map) o).containsKey("content")) {
-                        //map.put(field.getCode(), stringObjectMap.get(pathValue));
+//                        try {
-                        map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
+//                            return ((Map<String, String>) o).get("content");
-                    }
+//                        }
-                }else {
+//                        catch (ClassCastException e){
-                    if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
+//                            if(((Map<?, ?>) o).get("content") instanceof Integer) {
-                        String value = null;
+//                                return String.valueOf(((Map<?, ?>) o).get("content"));
-                        Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
+//                            }
-                        if (fieldObj != null){
+//                            return null;
-                            if (fieldObj instanceof Map){
+//                        }
-                                Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
+//                    }
-                                if(o instanceof String){
+//                }
-                                    value = (String)o;
+//            }
-                                }
+//        } else if (value instanceof Map) {
-                                else if(o instanceof Integer){
+//            String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
-                                    value = String.valueOf(o);
+//            return ((Map<String, String>)value).get(key);
-                                }
+//        }
-                            } else if (fieldObj instanceof List) {
+//        return value != null ? value.toString() : null;
-                                Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
+//    }
-                                if(o instanceof String){
-                                    value = (String)o;
-                                }
-                                else if(o instanceof Integer){
-                                    value = String.valueOf(o);
-                                }
-                            }
-                        }
-                        if (value != null){
-                            map.put(field.getCode(), value);
-                        }
-                    }
-                }
-            }
-            parsedData.add(map);
-        }
-        return parsedData;
-    }
-
-
-    private static String normalizeValue(Object value) {
-        if (value instanceof JSONArray) {
-            JSONArray jarr = (JSONArray) value;
-            if (jarr.get(0) instanceof String) {
-                return jarr.get(0).toString();
-            } else {
-                for (Object o : jarr) {
-                    if ((o instanceof Map) && ((Map) o).containsKey("content")) {
-                        try {
-                            return ((Map<String, String>) o).get("content");
-                        }
-                        catch (ClassCastException e){
-                            if(((Map<?, ?>) o).get("content") instanceof Integer) {
-                                return String.valueOf(((Map<?, ?>) o).get("content"));
-                            }
-                            return null;
-                        }
-                    }
-                }
-            }
-        } else if (value instanceof Map) {
-            String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
-            return ((Map<String, String>)value).get(key);
-        }
-        return value != null ? value.toString() : null;
-    }
 
 }

dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java

@@ -18,7 +18,6 @@ import eu.eudat.service.dmpblueprint.DmpBlueprintServiceImpl;
 import gr.cite.commons.web.authz.service.AuthorizationService;
 import gr.cite.tools.data.builder.BuilderFactory;
 import gr.cite.tools.data.deleter.DeleterFactory;
-import gr.cite.tools.data.query.QueryFactory;
 import gr.cite.tools.exception.MyApplicationException;
 import gr.cite.tools.exception.MyForbiddenException;
 import gr.cite.tools.exception.MyNotFoundException;

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java

@@ -1,22 +1,14 @@
 package eu.eudat.service.remotefetcher;
 
-import eu.eudat.commons.enums.ReferenceType;
+import eu.eudat.data.ReferenceTypeEntity;
-import eu.eudat.commons.exceptions.HugeResultSetException;
+import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
-import eu.eudat.service.remotefetcher.config.entities.GenericUrls;
 import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
-import gr.cite.tools.exception.MyNotFoundException;
+import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
+import eu.eudat.service.remotefetcher.models.ExternalDataResult;
 
 import java.util.List;
-import java.util.Map;
 
 public interface RemoteFetcherService {
-	List<Map<String, String>> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException;
+	ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy);
-
+	Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key);
-	GenericUrls getExternalUrls(ReferenceType referenceType);
-
-	Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException;
-
-	List<Map<String, String>> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls);
-
-	List<Map<String, Object>> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls);
 }

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java

@@ -5,19 +5,16 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.jayway.jsonpath.DocumentContext;
 import com.jayway.jsonpath.JsonPath;
-import eu.eudat.commons.enums.ReferenceType;
+import eu.eudat.commons.XmlHandlingService;
+import eu.eudat.commons.enums.ReferenceTypeSourceType;
 import eu.eudat.commons.exceptions.HugeResultSetException;
-import eu.eudat.service.remotefetcher.config.AuthenticationConfiguration;
+import eu.eudat.convention.ConventionService;
-import eu.eudat.service.remotefetcher.config.DataUrlConfiguration;
+import eu.eudat.service.remotefetcher.config.entities.*;
-import eu.eudat.service.remotefetcher.config.QueryConfig;
+import eu.eudat.service.remotefetcher.models.ExternalDataResult;
-import eu.eudat.service.remotefetcher.config.UrlConfiguration;
-import eu.eudat.service.remotefetcher.config.entities.GenericUrls;
-import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
 import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
 import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
-import gr.cite.tools.exception.MyNotFoundException;
+import gr.cite.tools.exception.MyApplicationException;
-import jakarta.xml.bind.JAXBContext;
+import net.minidev.json.JSONArray;
-import jakarta.xml.bind.Unmarshaller;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -26,13 +23,10 @@ import org.springframework.http.*;
 import org.springframework.http.client.reactive.ReactorClientHttpConnector;
 import org.springframework.http.codec.json.Jackson2JsonDecoder;
 import org.springframework.stereotype.Service;
-import org.springframework.web.client.RestTemplate;
 import org.springframework.web.reactive.function.client.WebClient;
 import reactor.netty.http.client.HttpClient;
 
 import java.io.File;
-import java.io.StringReader;
-import java.lang.reflect.Method;
 import java.nio.file.Paths;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -43,9 +37,13 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
 
     private WebClient webClient;
     private final ExternalUrlConfigProvider externalUrlConfigProvider;
+	private final ConventionService conventionService;
+    private final XmlHandlingService xmlHandlingService;
     @Autowired
-    public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider) {
+    public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, XmlHandlingService xmlHandlingService) {
 	    this.externalUrlConfigProvider = externalUrlConfigProvider;
+	    this.conventionService = conventionService;
+	    this.xmlHandlingService = xmlHandlingService;
     }
 
     private WebClient getWebClient(){
@@ -59,158 +57,96 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
         return webClient;
     }
 
-    @Override
-    public List<Map<String, String>> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException {
-        FetchStrategy fetchStrategy = null;
-        GenericUrls exGenericUrls = this.getExternalUrls(referenceType);
-        List<UrlConfiguration> urlConfigs =  key != null && !key.isEmpty() ? exGenericUrls.getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
-                : exGenericUrls.getUrls();
 
-        List<Map<String, String>> results = getAll(urlConfigs, fetchStrategy, externalReferenceCriteria);
+
-        for (Map<String, String> result: results) {
+    @Override
-            result.put("referenceType", referenceType.name());
+    public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy)  {
+        List<SourceBaseConfiguration> apiSourcesToUse =  sources.stream().map(x -> (SourceBaseConfiguration)x).toList();
+        if (!this.conventionService.isNullOrEmpty(key)){
+            apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).map(x -> (SourceBaseConfiguration)x).toList();
         }
-        return results;
+        if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult();
+
+        apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal));
+
+        return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria);
     }
 
     @Override
-    public GenericUrls getExternalUrls(ReferenceType referenceType)  {
+    public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
-	    return switch (referenceType) {
+        return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size();
-		    case Taxonomies -> this.externalUrlConfigProvider.getExternalUrls().getTaxonomies();
-		    case Licenses -> this.externalUrlConfigProvider.getExternalUrls().getLicenses();
-		    case Publications -> this.externalUrlConfigProvider.getExternalUrls().getPublications();
-		    case Journals -> this.externalUrlConfigProvider.getExternalUrls().getJournals();
-		    case PubRepositories -> this.externalUrlConfigProvider.getExternalUrls().getPubRepositories();
-		    case DataRepositories -> this.externalUrlConfigProvider.getExternalUrls().getRepositories();
-		    case Registries -> this.externalUrlConfigProvider.getExternalUrls().getRegistries();
-		    case Services -> this.externalUrlConfigProvider.getExternalUrls().getServices();
-		    case Grants -> this.externalUrlConfigProvider.getExternalUrls().getGrants();
-		    case Organizations -> this.externalUrlConfigProvider.getExternalUrls().getOrganisations();
-		    case Datasets -> this.externalUrlConfigProvider.getExternalUrls().getDatasets();
-		    case Funder -> this.externalUrlConfigProvider.getExternalUrls().getFunders();
-		    case Project -> this.externalUrlConfigProvider.getExternalUrls().getProjects();
-		    case Researcher -> this.externalUrlConfigProvider.getExternalUrls().getResearchers();
-		    default -> throw new IllegalArgumentException("Type not found" + referenceType);
-	    };
     }
 
-    @Override
+    private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
-    public Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException {
-    	List<UrlConfiguration> urlConfigs =
-                key != null && !key.isEmpty() ? this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
-                        : this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls();
-       FetchStrategy fetchStrategy = this.externalUrlConfigProvider.getExternalUrls().getValidations().getFetchMode();
-       List<Map<String, String>> data = this.getAll(urlConfigs, fetchStrategy, externalReferenceCriteria);
-        return data.size();
-    }
 
-    @Override
+        ExternalDataResult results = new ExternalDataResult();
-    public List<Map<String, String>> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) {
-        List<UrlConfiguration> urlConfigurations = genericUrls.getUrls();
-        FetchStrategy fetchStrategy = genericUrls.getFetchMode();
-        return getAll(urlConfigurations, fetchStrategy, externalReferenceCriteria);
-    }
 
-    @Override
+	    if (this.conventionService.isListNullOrEmpty(sources)) return new ExternalDataResult();
-    public List<Map<String, Object>> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) {
-        List<UrlConfiguration> urlConfigurations = genericUrls.getUrls();
-        return getAllWithData(urlConfigurations, externalReferenceCriteria);
-    }
 
-    private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
+	    for (SourceBaseConfiguration source : sources) {
-
+		    if (source.getType() == null || source.getType().equals(ReferenceTypeSourceType.API)) {
-        List<Map<String, String>> results = new LinkedList<>();
-
-        if (urlConfigs == null || urlConfigs.isEmpty()) {
-            return results;
-        }
-//            throw new MyNotFoundException("No Repository urls found in configuration");
-
-        urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
-	    for (UrlConfiguration urlConfiguration : urlConfigs) {
-		    applyFunderQuery(urlConfiguration, externalReferenceCriteria);
-		    if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
 			    try {
+                    SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource = (SourceExternalApiConfiguration)source;
+                    this.applyFunderToQuery(apiSource, externalReferenceCriteria);
+
                     String auth = null;
-				    if (urlConfiguration.getAuth() != null) {
+				    if (apiSource.getAuth() != null) {
-					    auth = this.getAuthentication(urlConfiguration.getAuth());
+					    auth = this.buildAuthentication(apiSource.getAuth());
 				    }
-				    results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalReferenceCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getFilterType(), urlConfiguration.getQueries(), auth));
+				    results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth));
 			    } catch (Exception e) {
 				    logger.error(e.getLocalizedMessage(), e);
 			    }
-		    } else if (urlConfiguration.getType() != null && urlConfiguration.getType().equals("Internal")) {
+		    } else if (source.getType() != null &&  source.getType().equals(ReferenceTypeSourceType.STATIC)) {
-			    results.addAll(getAllResultsFromMockUpJson(urlConfiguration.getUrl(), externalReferenceCriteria.getLike()));
+                SourceStaticOptionConfiguration<StaticOption> staticSource = (SourceStaticOptionConfiguration)source;
+			    results.addAll(getAllResultsFromMockUpJson(staticSource.getKey(), externalReferenceCriteria.getLike())); //TODO:
 		    }
 	    }
-       /* for (UrlConfiguration urlConfig : urlConfigs) {
-            ifFunderQueryExist(urlConfig, externalUrlCriteria);
-            if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
-                results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType()));
-            } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
-                results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
-            }
-        }*/
         return results;
     }
 
-    private String getAuthentication(AuthenticationConfiguration authenticationConfiguration) {
+    private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) {
-        HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod());
+        HttpMethod method;
+        switch (authenticationConfiguration.getAuthMethod()) {
+            case GET -> method = HttpMethod.GET;
+            case POST -> method =HttpMethod.POST;
+            default -> throw new MyApplicationException("unrecognized type " + authenticationConfiguration.getAuthMethod());
+        }
+        
         Map<String, Object> response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl())
                 .contentType(MediaType.APPLICATION_JSON)
                 .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
                 .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
                 })).block();
+        if (response == null) throw new MyApplicationException("Authentication " + authenticationConfiguration.getAuthUrl() + " failed");
 
-
+        return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null);
-
-        return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
     }
 
-    private List<Map<String, Object>> getAllWithData(List<UrlConfiguration> urlConfigs, ExternalReferenceCriteria externalReferenceCriteria) {
 
-        List<Map<String, Object>> results = new LinkedList<>();
+    private void applyFunderToQuery(SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) {
-
+        //TODO new reference logic
-        if (urlConfigs == null || urlConfigs.isEmpty()) {
+//	    if (apiSource.getFunderQuery() != null) {
-            return results;
+//		    if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) {
+//			    apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery()));
+//		    }
+//		    else {
+//			    apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", ""));
+//		    }
+//	    }
     }
 
-        urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
+    private String replaceLookupFieldQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) {
-        urlConfigs.forEach(urlConfiguration -> {
-            applyFunderQuery(urlConfiguration, externalReferenceCriteria);
-            if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
-                try {
-                    results.addAll(getAllResultsFromUrlWithData(urlConfiguration.getUrl(), urlConfiguration.getData(), externalReferenceCriteria,  urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getQueries()));
-                } catch (Exception e) {
-                    logger.error(e.getLocalizedMessage(), e);
-                }
-            }
-        });
-        return results;
-
-    }
-
-    private void applyFunderQuery(UrlConfiguration urlConfiguration, ExternalReferenceCriteria externalReferenceCriteria) {
-        if (urlConfiguration.getFunderQuery() != null) {
-            if (externalReferenceCriteria.getFunderId() != null && !urlConfiguration.getFunderQuery().startsWith("dmp:")) {
-                urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", urlConfiguration.getFunderQuery()));
-            }
-            else {
-                urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", ""));
-            }
-        }
-    }
-
-    private String calculateQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) {
         String finalQuery = "";
-        QueryConfig queryConfig = queryConfigs.stream().filter(queryConfigl -> externalReferenceCriteria.getLike().matches(queryConfigl.getCondition()))
+        if (this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike())) return finalQuery;
+        QueryConfig queryConfig = queryConfigs.stream().filter(x -> !this.conventionService.isNullOrEmpty(x.getCondition()) && externalReferenceCriteria.getLike().matches(x.getCondition()))
                 .min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null);
         if (queryConfig != null) {
             if (queryConfig.getSeparator() != null) {
                 String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator());
                 finalQuery = queryConfig.getValue();
                 for (int i = 0; i < likes.length; i++) {
-                    finalQuery = finalQuery.replaceAll("\\{like" + (i+1) + "}", likes[i]);
+                    finalQuery = finalQuery.replaceAll("\\{like" + (i + 1) + "}", likes[i]);
                 }
             } else {
                 finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike());
@@ -220,15 +156,17 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
         return finalQuery;
     }
 
-    protected String replaceCriteriaOnUrl(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) {
+    protected String replaceLookupFields(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) {
+        if (!this.conventionService.isNullOrEmpty(path)) return path;
         String completedPath = path;
+        
         if (externalReferenceCriteria.getLike() != null) {
-            if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().equals("")) {
+            if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().isEmpty()) {
                 completedPath = completedPath.replaceAll("\\{like}", "*");
                 completedPath = completedPath.replaceAll("\\{query}", "*");
             } else {
                 if (completedPath.contains("{query}")) {
-                    completedPath = completedPath.replaceAll("\\{query}", this.calculateQuery(externalReferenceCriteria, queries));
+                    completedPath = completedPath.replaceAll("\\{query}", this.replaceLookupFieldQuery(externalReferenceCriteria, queries));
                 } else {
                     completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike());
                 }
@@ -236,250 +174,186 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
         } else {
             completedPath = completedPath.replace("{like}", "");
         }
-        if (externalReferenceCriteria.getFunderId() != null) {
+        if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getFunderId())) {
-            String funderPrefix = externalReferenceCriteria.getFunderId().split(":")[0];
+            String[] founderParts = externalReferenceCriteria.getFunderId().split(":");
+            String funderPrefix = founderParts.length > 0 ? founderParts[0] : "";
             String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", "");
-            if (funderId.toCharArray()[0] == ':') {
+            if (!funderId.isEmpty() && funderId.toCharArray()[0] == ':') {
                 funderId = externalReferenceCriteria.getFunderId();
             }
-			/*
-			  try { funderId = URLEncoder.encode(funderId, "UTF-8"); } catch
-			  (UnsupportedEncodingException e) { logger.error(e.getMessage(), e); }
-			 */
             completedPath = completedPath.replace("{funderId}", funderId);
-        }
+        } else if(completedPath.contains("{funderId}")){
-        else if(completedPath.contains("{funderId}")){
             logger.warn("FunderId is null.");
             completedPath = completedPath.replace("{funderId}", " ");
         }
-        if (externalReferenceCriteria.getPage() != null) {
+        
-            completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage());
+        if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPage()))  completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage());
-        } else {
+        else  if (!this.conventionService.isNullOrEmpty(firstPage))  completedPath = completedPath.replace("{page}", firstPage);
-            if (firstPage != null) {
+        else completedPath = completedPath.replace("{page}", "1");
-                completedPath = completedPath.replace("{page}", firstPage);
+
-            } else {
+        completedPath = completedPath.replace("{pageSize}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPageSize()) ? externalReferenceCriteria.getPageSize() : "60");
-                completedPath = completedPath.replace("{page}", "1");
+        completedPath = completedPath.replace("{host}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getHost()) ? externalReferenceCriteria.getHost() : "");
-            }
+        completedPath = completedPath.replace("{path}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPath()) ? externalReferenceCriteria.getPath() : "");
-        }
+        
-        if (externalReferenceCriteria.getPageSize() != null) {
+        
-            completedPath = completedPath.replace("{pageSize}", externalReferenceCriteria.getPageSize());
-        } else {
-            completedPath = completedPath.replace("{pageSize}", "60");
-        }
-        if (externalReferenceCriteria.getHost() != null) {
-            completedPath = completedPath.replace("{host}", externalReferenceCriteria.getHost());
-        } else {
-            completedPath = completedPath.replace("{host}", "");
-        }
-        if (externalReferenceCriteria.getPath() != null) {
-            completedPath = completedPath.replace("{path}", externalReferenceCriteria.getPath());
-        } else {
-            completedPath = completedPath.replace("{path}", "");
-        }
         return completedPath;
     }
 
-    private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalReferenceCriteria externalReferenceCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType, String filterType, List<QueryConfig> queries, String auth) throws Exception {
+    private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
-        Set<Integer> pages = new HashSet<>();
+        String replacedPath = replaceLookupFields(apiSource.getUrl(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
+        String replacedBody = replaceLookupFields(apiSource.getRequestBody(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
 
-        String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries);
+        ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth);
-        String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries);
+        if(externalDataResult != null) {
-
+            if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
-        ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth);
+                externalDataResult.setResults(externalDataResult.getResults().stream()
-        if(externalRefernceResult != null) {
-            if (filterType != null && filterType.equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
-                externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
                         .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
                         .collect(Collectors.toList()));
             }
-            if (fetchStrategy == FetchStrategy.FIRST)
+            externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put("tag", apiSource.getLabel())).peek(x -> x.put("key", apiSource.getKey())).toList());
-                return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
+            if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult;
-
-            if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set
-                for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++)
-                    pages.add(i);
 
             Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
-            if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults))
+            if (externalDataResult.getResults().size() > maxResults)
-                throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query");
+                throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query");
 
-            Optional<ExternalRefernceResult> optionalResults = pages.parallelStream()
+            return externalDataResult;
-                    .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth))
-                    .filter(Objects::nonNull)
-                    .reduce((result1, result2) -> {
-                        result1.getResults().addAll(result2.getResults());
-                        return result1;
-                    });
-            ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new);
-            remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults());
-
-            return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
         }
         else {
-            return new LinkedList<>();
+            return new ExternalDataResult();
         }
     }
 
-    private List<Map<String, Object>> getAllResultsFromUrlWithData(String path, final DataUrlConfiguration jsonDataPath, ExternalReferenceCriteria externalReferenceCriteria, String contentType, String firstPage, String requestBody, String requestType, List<QueryConfig> queries) {
+    protected ExternalDataResult getExternalDataResults(String urlString, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig>  apiSource, String requestBody, String auth) {
-
-        String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries);
-        String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries);
 
         try {
-            RestTemplate restTemplate = new RestTemplate();
-            HttpHeaders headers = new HttpHeaders();
-            HttpEntity<JsonNode> entity;
-            ResponseEntity<String> response;
-            if (contentType != null && !contentType.isEmpty()) {
-                headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
-                headers.setContentType(MediaType.valueOf(contentType));
-            }
-            JsonNode jsonBody = new ObjectMapper().readTree(replacedBody);
-            entity = new HttpEntity<>(jsonBody, headers);
-
-            response = restTemplate.exchange(replacedPath, HttpMethod.valueOf(requestType), entity, String.class);
-            if (response.getStatusCode() == HttpStatus.OK) {
-                if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
-                    DocumentContext jsonContext = JsonPath.parse(response.getBody());
-                    return jsonContext.read(jsonDataPath.getPath());
-                }
-            }
-        }
-        catch (Exception exception) {
-            logger.error(exception.getMessage(), exception);
-        }
-
-        return new LinkedList<>();
-    }
-
-
-    protected ExternalRefernceResult getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType, String auth) {
-
-        try {
-        	//RestTemplate restTemplate = new RestTemplate(new SimpleClientHttpRequestFactory());
-        	//HttpHeaders headers = new HttpHeaders();
-        	//HttpEntity<JsonNode> entity;
-        	ResponseEntity<String> response;
-			/*
-			 * URL url = new URL(urlString.replaceAll(" ", "%20"));
-			 * 
-			 * HttpURLConnection con = (HttpURLConnection) url.openConnection();
-			 * con.setRequestMethod("GET");
-			 */
-           /* if (contentType != null && !contentType.isEmpty()) {
-               headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
-               headers.setContentType(MediaType.valueOf(contentType));
-            }
-            if (auth != null) {
-                headers.set("Authorization", auth);
-            }*/
             JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
-//            entity = new HttpEntity<>(jsonBody, headers);
+            HttpMethod method;
+            switch (apiSource.getHttpMethod()) {
+                case GET -> method = HttpMethod.GET;
+                case POST -> method =HttpMethod.POST;
+                default -> throw new MyApplicationException("unrecognized type " + apiSource.getHttpMethod());
+            }
             
-
+            ResponseEntity<String> response = this.getWebClient().method(method).uri(urlString).headers(httpHeaders -> {
-            response = this.getWebClient().method(HttpMethod.valueOf(requestType)).uri(urlString).headers(httpHeaders -> {
+                if (this.conventionService.isNullOrEmpty(apiSource.getContentType())) {
-                if (contentType != null && !contentType.isEmpty()) {
+                    httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(apiSource.getContentType())));
-                    httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
+                    httpHeaders.setContentType(MediaType.valueOf(apiSource.getContentType()));
-                    httpHeaders.setContentType(MediaType.valueOf(contentType));
                 }
                 if (auth != null) {
                     httpHeaders.set("Authorization", auth);
                 }
             }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
-            //response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class);
+            if (response == null || !response.getStatusCode().isSameCodeAs(HttpStatus.OK) || !response.hasBody() || response.getBody()  == null) return null;
-            if (response.getStatusCode() == HttpStatus.OK) { // success
+            
             //do here all the parsing
-                ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult();
+            List<String> responseContentTypeHeader = response.getHeaders().getOrDefault("Content-Type", null);
-                if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
+            String responseContentType =  !this.conventionService.isListNullOrEmpty(responseContentTypeHeader)  && responseContentTypeHeader.getFirst() != null ? responseContentTypeHeader.getFirst() : "";
+            
+            if (responseContentType.contains("json") ) {
                 DocumentContext jsonContext = JsonPath.parse(response.getBody());
-
+                return this.parseData(jsonContext, apiSource.getResults());
-                    if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) {
-                        externalRefernceResult = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType, auth);
-                    } else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) {
-                        externalRefernceResult = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath);
             } else {
-                        externalRefernceResult = RemoteFetcherUtils.getFromJson(jsonContext, jsonDataPath);
+                throw new MyApplicationException("Unsupported response type" + responseContentType);
-                    }
-                    externalRefernceResult.setResults(externalRefernceResult.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
-                                                     .collect(Collectors.toList()));
-                }
-                else if (response.getHeaders().get("Content-Type").get(0).contains("xml")) {
-                    Class<?> aClass = Class.forName(jsonDataPath.getParseClass());
-                    JAXBContext jaxbContext = JAXBContext.newInstance(aClass);
-                    Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
-                    StringReader stringReader = new StringReader(response.getBody());
-                    Object data = unmarshaller.unmarshal(stringReader);
-                    Method reader = null;
-                    if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) {
-                        String camelCaseGetter = jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty() ? "get" + jsonDataPath.getParseField().substring(0, 1).toUpperCase() + jsonDataPath.getParseField().substring(1) : "";
-                        reader = aClass.getMethod(camelCaseGetter);
-                    }
-                    ObjectMapper objectMapper = new ObjectMapper();
-                    List<Map<String, String>> values = new ArrayList<>();
-                    int max = 1;
-                    if (reader != null) {
-                        Object invokedField = reader.invoke(data);
-                        if (invokedField instanceof Collection) {
-                            max = ((Collection) invokedField).size();
-                        }
-                    }
-                    for (int i = 0; i< max; i++) {
-                        Object value;
-                        if (reader != null) {
-                            Object invokedField = reader.invoke(data);
-                            if (invokedField instanceof Collection) {
-                                value = ((Collection) invokedField).toArray()[i];
-                            } else {
-                                value = invokedField;
-                            }
-                        } else {
-                            value = data;
-                        }
-                        Map<String, String> map = objectMapper.convertValue(value, Map.class);
-                        if (jsonDataPath.getMergedFields() != null && !jsonDataPath.getMergedFields().isEmpty() && jsonDataPath.getMergedFieldName() != null && !jsonDataPath.getMergedFieldName().isEmpty()) {
-                            Map<String, String> finalMap = new HashMap<>();
-                            for (Map.Entry<String, String> entry : map.entrySet()) {
-                                if (jsonDataPath.getMergedFields().contains(entry.getKey())) {
-                                    if (!finalMap.containsKey(jsonDataPath.getMergedFieldName())) {
-                                        finalMap.put(jsonDataPath.getMergedFieldName(), entry.getValue());
-                                    } else {
-                                        finalMap.put(jsonDataPath.getMergedFieldName(), finalMap.get(jsonDataPath.getMergedFieldName()) + " " + entry.getValue());
-                                    }
-                                } else {
-                                    finalMap.put(entry.getKey(), entry.getValue());
-                                }
-                            }
-                            values.add(finalMap);
-                        } else {
-                            values.add(map);
-                        }
-                    }
-                    externalRefernceResult = new ExternalRefernceResult(values, new HashMap<>(1, 1));
             }
 
-                if (externalRefernceResult.getPagination().isEmpty()) {
-                    externalRefernceResult.getPagination().put("pages", 1);
-                    externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size());
-                }
-                return externalRefernceResult;
-            }
         } catch (Exception exception) {
             logger.error(exception.getMessage(), exception);
-        } //maybe print smth...
+        } 
 
         return null;
     }
 
-    private List<Map<String, String>> getAllResultsFromMockUpJson(String path, String query) {
+    private ExternalDataResult parseData (DocumentContext jsonContext, ResultsConfiguration<ResultFieldsMappingConfiguration> resultsConfigurationEntity) {
+        ExternalDataResult result = new ExternalDataResult();
+        if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult();
+        List<Map<String, Object>> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath());
+
+        result.setRawData(rawData);
+        
+        if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult();
+        
+        List<Map<String, String>> parsedData = new ArrayList<>();
+        for (Map<String, Object> stringObjectMap: rawData){
+            Map<String, String> map = new HashMap<>();
+            for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()){
+                String pathValue = field.getResponsePath();
+                if (!pathValue.contains(".")){
+                    if (stringObjectMap.containsKey(pathValue)) {
+                        map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
+                    }
+                }else {
+                    if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
+                        String value = null;
+                        Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
+                        if (fieldObj != null){
+                            if (fieldObj instanceof Map){
+                                Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
+                                if(o instanceof String){
+                                    value = (String)o;
+                                }
+                                else if(o instanceof Integer){
+                                    value = String.valueOf(o);
+                                }
+                            } else if (fieldObj instanceof List) {
+                                Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
+                                if(o instanceof String){
+                                    value = (String)o;
+                                }
+                                else if(o instanceof Integer){
+                                    value = String.valueOf(o);
+                                }
+                            }
+                        }
+                        if (value != null){
+                            map.put(field.getCode(), value);
+                        }
+                    }
+                }
+            }
+            parsedData.add(map);
+        }
+        result.setResults(parsedData);
+        return result;
+    }
+
+    private static String normalizeValue(Object value) {
+        if (value instanceof JSONArray jsonArray) {
+            
+	        if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) {
+                return jsonArray.getFirst().toString();
+            } else {
+                for (Object o : jsonArray) {
+                    if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) {
+                        try {
+                            return ((Map<String, String>) o).get("content");
+                        }
+                        catch (ClassCastException e){
+                            if(((Map<?, ?>) o).get("content") instanceof Integer) {
+                                return String.valueOf(((Map<?, ?>) o).get("content"));
+                            }
+                            return null;
+                        }
+                    }
+                }
+            }
+        } else if (value instanceof Map) {
+            String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
+            return ((Map<String, String>)value).get(key);
+        }
+        return value != null ? value.toString() : null;
+    }
+    
+    private ExternalDataResult getAllResultsFromMockUpJson(String path, String query) {
         List<Map<String, String>> internalResults;
         try {
             String filePath = Paths.get(path).toUri().toURL().toString();
             ObjectMapper mapper = new ObjectMapper();
             internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
-            return searchListMap(internalResults, query);
+            return new ExternalDataResult(searchListMap(internalResults, query));
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
-            return new LinkedList<>();
+            return new ExternalDataResult();
         }
     }
 
@@ -497,25 +371,6 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
         return list;
     }
 
-    private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) {
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) {
-            if(dataUrlConfiguration.getFieldsUrlConfiguration().getPid() == null)
-                return "pid";
-            else
-                return "originalId";
-        }
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getPid() != null && key.equals("pid")) return "pid";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getPidTypeField() != null && key.equals("pidTypeField")) return "pidTypeField";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getPath() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getPath().replace("'",""))) return "path";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getHost() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getHost().replace("'",""))) return "host";
-        return null;
-    }
-
     private String parseBodyString(String bodyString) {
         String finalBodyString = bodyString;
         if (bodyString.contains("{env:")) {

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherUtils.java

@@ -1,152 +0,0 @@
-package eu.eudat.service.remotefetcher;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.jayway.jsonpath.DocumentContext;
-import eu.eudat.service.remotefetcher.config.DataUrlConfiguration;
-import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
-import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
-import net.minidev.json.JSONArray;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.*;
-import java.util.stream.Collectors;
-
-public class RemoteFetcherUtils {
-    private final static Logger logger = LoggerFactory.getLogger(RemoteFetcherUtils.class);
-    private static final ObjectMapper mapper = new ObjectMapper();
-
-    public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
-        return new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
-                new HashMap<>(1, 1));
-    }
-
-    public static ExternalRefernceResult getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcherServiceImpl remoteFetcherService, String requestBody, String requestType, String auth) {
-        ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
-                new HashMap<>(1, 1));
-
-        List<Map<String, String>> multiResults = externalRefernceResult.getResults().stream().map(result -> {
-            ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria();
-            externalReferenceCriteria.setPath(result.get("path"));
-            externalReferenceCriteria.setHost(result.get("host"));
-            String replacedPath = remoteFetcherService.replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalReferenceCriteria, jsonDataPath.getUrlConfiguration().getFirstpage(), jsonDataPath.getUrlConfiguration().getQueries());
-            return remoteFetcherService.getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType, auth);
-        }).filter(Objects::nonNull).map(externalRefernceResult1 -> externalRefernceResult1.getResults().get(0)).collect(Collectors.toList());
-        return new ExternalRefernceResult(multiResults, new HashMap<>(1, 1));
-    }
-
-    public static ExternalRefernceResult getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
-        ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
-                new HashMap<>(1, 1));
-        externalRefernceResult.getResults().stream().forEach(entry -> {
-            String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
-            entry.put("name", name);
-            entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", ""));
-            entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
-        });
-        return externalRefernceResult;
-    }
-
-    private static List<Map<String, String>> parseData (DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
-        List <Map<String, Object>> rawData = jsonContext.read(jsonDataPath.getPath());
-        List<Map<String, String>> parsedData = new ArrayList<>();
-        rawData.forEach(stringObjectMap -> {
-            parsedData.add(new LinkedHashMap<>());
-            Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredFields()).forEach(field -> {
-                String getterMethodName = "get" + field.getName().substring(0, 1).toUpperCase(Locale.ROOT) + field.getName().substring(1);
-                Method getterMethod = Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredMethods()).filter(method -> method.getName().equals(getterMethodName)).collect(Collectors.toList()).get(0);
-                try {
-                    String value = ((String) getterMethod.invoke(jsonDataPath.getFieldsUrlConfiguration()));
-                    if (value != null) {
-                        if (field.getName().equals("pid") || field.getName().equals("pidTypeField")) {
-                            String pid = null;
-                            Object pidObj = stringObjectMap.get(value.split("\\.")[0]);
-                            if(pidObj != null){
-                                if(pidObj instanceof Map){
-                                    Object o = ((Map<String, Object>) pidObj).get(value.split("\\.")[1]);
-                                    if(o instanceof String){
-                                        pid = (String)o;
-                                    }
-                                    else if(o instanceof Integer){
-                                        pid = String.valueOf(o);
-                                    }
-                                }
-                                else if(pidObj instanceof List){
-                                    Object o = ((List<Map<String,?>>) pidObj).get(0).get(value.split("\\.")[1]);
-                                    if(o instanceof String){
-                                        pid = (String)o;
-                                    }
-                                    else if(o instanceof Integer){
-                                        pid = String.valueOf(o);
-                                    }
-                                }
-                            }
-                            if(pid != null) {
-                                if ((field.getName().equals("pid"))){
-                                    parsedData.get(parsedData.size() - 1).put("pid", pid);
-                                }
-                                else{
-                                    parsedData.get(parsedData.size() - 1).put("pidTypeField", pid);
-                                }
-                            }
-                        } else {
-                            value = value.replace("'", "");
-                            if (value.contains(".")) {
-                                String[] parts = value.split("\\.");
-                                Map<String, Object> tempMap = stringObjectMap;
-                                for (int i = 0; i < parts.length; i++) {
-                                    if (tempMap.containsKey(parts[i])) {
-                                        if (i + 1 < parts.length) {
-                                            tempMap = (Map<String, Object>) tempMap.get(parts[i]);
-                                        } else {
-                                            parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(tempMap.get(parts[i]), (field.getName().equals("types") || field.getName().equals("uri"))));
-                                        }
-                                    }
-                                }
-                            } else {
-                                if (stringObjectMap.containsKey(value)) {
-                                    parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(stringObjectMap.get(value), (field.getName().equals("types") || field.getName().equals("uri"))));
-                                }
-                            }
-                        }
-                    }
-                } catch (IllegalAccessException | InvocationTargetException e) {
-                    logger.error(e.getLocalizedMessage(), e);
-                }
-            });
-        });
-        return parsedData;
-    }
-
-    private static String normalizeValue(Object value, boolean jsonString) {
-        if (value instanceof JSONArray) {
-            if (jsonString) {
-                return ((JSONArray)value).toJSONString();
-            }
-            JSONArray jarr = (JSONArray) value;
-            if (jarr.get(0) instanceof String) {
-                return jarr.get(0).toString();
-            } else {
-                for (Object o : jarr) {
-                    if ((o instanceof Map) && ((Map) o).containsKey("content")) {
-                        try {
-                            return ((Map<String, String>) o).get("content");
-                        }
-                        catch (ClassCastException e){
-                            if(((Map<?, ?>) o).get("content") instanceof Integer) {
-                                return String.valueOf(((Map<?, ?>) o).get("content"));
-                            }
-                            return null;
-                        }
-                    }
-                }
-            }
-        } else if (value instanceof Map) {
-            String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
-            return ((Map<String, String>)value).get(key);
-        }
-        return value != null ? value.toString() : null;
-    }
-}

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/AuthenticationConfiguration.java

@@ -1,57 +0,0 @@
-package eu.eudat.service.remotefetcher.config;
-
-import jakarta.xml.bind.annotation.XmlElement;
-
-public class AuthenticationConfiguration {
-
-    private String authUrl;
-    private String authMethod = "GET";
-    private String authTokenPath;
-    private String authRequestBody;
-    private String type;
-
-    public String getAuthUrl() {
-        return authUrl;
-    }
-
-    @XmlElement(name = "authUrl")
-    public void setAuthUrl(String authUrl) {
-        this.authUrl = authUrl;
-    }
-
-    public String getAuthMethod() {
-        return authMethod;
-    }
-
-    @XmlElement(name = "authUrlMethod")
-    public void setAuthMethod(String authMethod) {
-        this.authMethod = authMethod;
-    }
-
-    public String getAuthTokenPath() {
-        return authTokenPath;
-    }
-
-    @XmlElement(name = "authTokenJpath")
-    public void setAuthTokenPath(String authTokenPath) {
-        this.authTokenPath = authTokenPath;
-    }
-
-    public String getAuthRequestBody() {
-        return authRequestBody;
-    }
-
-    @XmlElement(name = "authUrlBody")
-    public void setAuthRequestBody(String authRequestBody) {
-        this.authRequestBody = authRequestBody;
-    }
-
-    public String getType() {
-        return type;
-    }
-
-    @XmlElement(name = "authType")
-    public void setType(String type) {
-        this.type = type;
-    }
-}

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataFieldsUrlConfiguration.java

@@ -1,134 +0,0 @@
-package eu.eudat.service.remotefetcher.config;
-
-import jakarta.xml.bind.annotation.XmlElement;
-
-/**
- * Created by ikalyvas on 6/29/2018.
- */
-public class DataFieldsUrlConfiguration {
-    private String id;
-    private String name;
-    private String pid;
-    private String pidTypeField;
-    private String uri;
-    private String description;
-    private String source;
-    private String count;
-    private String path;
-    private String host;
-    private String types;
-    private String firstName;
-    private String lastName;
-
-    public String getId() {
-        return id;
-    }
-
-    @XmlElement(name = "id")
-    public void setId(String id) {
-        this.id = id;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    @XmlElement(name = "name")
-    public void setName(String name) {
-        this.name = name;
-    }
-
-    public String getPid() {
-        return pid;
-    }
-
-    @XmlElement(name = "pid")
-    public void setPid(String pid) {
-        this.pid = pid;
-    }
-
-    public String getPidTypeField() {
-        return pidTypeField;
-    }
-
-    @XmlElement(name = "pidTypeField")
-    public void setPidTypeField(String pidTypeField) {
-        this.pidTypeField = pidTypeField;
-    }
-
-    public String getUri() {
-        return uri;
-    }
-
-    @XmlElement(name = "uri")
-    public void setUri(String uri) {
-        this.uri = uri;
-    }
-
-    public String getDescription() {
-        return description;
-    }
-
-    @XmlElement(name = "description")
-    public void setDescription(String description) {
-        this.description = description;
-    }
-
-    public String getSource() {
-        return source;
-    }
-
-    @XmlElement(name = "source")
-    public void setSource(String source) {
-        this.source = source;
-    }
-
-    public String getCount() {
-        return count;
-    }
-
-    @XmlElement(name = "count")
-    public void setCount(String count) {
-        this.count = count;
-    }
-
-    public String getPath() {
-        return path;
-    }
-    @XmlElement(name = "path")
-    public void setPath(String path) {
-        this.path = path;
-    }
-
-    public String getHost() {
-        return host;
-    }
-    @XmlElement(name = "host")
-    public void setHost(String host) {
-        this.host = host;
-    }
-
-    @XmlElement(name = "types")
-    public String getTypes() {
-        return types;
-    }
-    public void setTypes(String types) {
-        this.types = types;
-    }
-
-    @XmlElement(name = "firstName")
-    public String getFirstName() {
-        return firstName;
-    }
-    public void setFirstName(String firstName) {
-        this.firstName = firstName;
-    }
-
-    @XmlElement(name = "lastName")
-    public String getLastName() {
-        return lastName;
-    }
-    public void setLastName(String lastName) {
-        this.lastName = lastName;
-    }
-}