Merge branch 'Development' of https://gitlab.eudat.eu/dmp/OpenAIRE-EUDAT-DMP-service-pilot into Development

dmp-backend/web/src/main/java/eu/eudat/controllers/DMPs.java

@@ -110,7 +110,7 @@ public class DMPs extends BaseController {
     @RequestMapping(method = RequestMethod.GET, value = {"rda/{id}"})
     public @ResponseBody
     ResponseEntity getRDAJsonDocument(@PathVariable String id, @ClaimedAuthorities(claims = {Authorities.ADMIN, Authorities.MANAGER, Authorities.USER, Authorities.ANONYMOUS}) Principal principal) throws IOException {
-        return this.dataManagementPlanManager.getRDAJsonDocument(id, principal);
+        return this.dataManagementPlanManager.getRDAJsonDocument(id, datasetManager, principal);
     }
 
     @RequestMapping(method = RequestMethod.GET, value = {"/overview/{id}"})

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java

@@ -1,5 +1,6 @@
 package eu.eudat.logic.managers;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.configurations.dynamicgrant.DynamicGrantConfiguration;
 import eu.eudat.configurations.dynamicgrant.entities.Property;
@@ -944,13 +945,14 @@ public class DataManagementPlanManager {
         return fileEnvelope;
     }
 
-    public ResponseEntity<byte[]> getRDAJsonDocument(String id, Principal principal) throws IOException {
+    public ResponseEntity<byte[]> getRDAJsonDocument(String id, DatasetManager datasetManager, Principal principal) throws IOException {
         eu.eudat.data.entities.DMP dmp = databaseRepository.getDmpDao().find(UUID.fromString(id));
-		if (!dmp.isPublic() && dmp.getUsers().stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()).collect(Collectors.toList()).size() == 0)
+		if (!dmp.isPublic() && dmp.getUsers().stream().noneMatch(userInfo -> userInfo.getUser().getId() == principal.getId()))
 			throw new UnauthorisedException();
-        RDAExportModel rdaExportModel = new RDAExportModel().fromDataModel(dmp);
+        RDAExportModel rdaExportModel = new RDAExportModel().fromDataModel(dmp, datasetManager);
 
         ObjectMapper mapper = new ObjectMapper();
+        mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
         String fileName = dmp.getLabel();
         fileName = fileName.replaceAll("[^a-zA-Z0-9+ ]", "");
         File file = new File(fileName);

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java

@@ -172,7 +172,7 @@ public class DatasetManager {
         return dataTable;
     }
 
-    public DatasetWizardModel getSingle(String id) throws InstantiationException, IllegalAccessException, IOException {
+    public DatasetWizardModel getSingle(String id) {
         DatasetWizardModel dataset = new DatasetWizardModel();
         eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
         eu.eudat.elastic.entities.Dataset datasetElastic;

dmp-backend/web/src/main/java/eu/eudat/models/data/rda/DatasetMetadataIdRDAExportModel.java

@@ -1,31 +0,0 @@
-package eu.eudat.models.data.rda;
-
-public class DatasetMetadataIdRDAExportModel {
-	private String metadata_id;
-	private String metadata_id_type;
-
-	public String getMetadata_id() {
-		return metadata_id;
-	}
-	public void setMetadata_id(String metadata_id) {
-		this.metadata_id = metadata_id;
-	}
-
-	public String getMetadata_id_type() {
-		return metadata_id_type;
-	}
-	public void setMetadata_id_type(String metadata_id_type) {
-		this.metadata_id_type = metadata_id_type;
-	}
-
-
-	public DatasetMetadataIdRDAExportModel fromDataModel(String key, Object value) {
-		DatasetMetadataIdRDAExportModel metadataIdRDAExportModel = new DatasetMetadataIdRDAExportModel();
-		if (key.contains(".metadata_id"))
-			metadataIdRDAExportModel.setMetadata_id(value.toString());
-		else if (key.contains(".metadata_id_type"))
-			metadataIdRDAExportModel.setMetadata_id_type(value.toString());
-
-		return metadataIdRDAExportModel;
-	}
-}

dmp-backend/web/src/main/java/eu/eudat/models/data/rda/DatasetMetadataRDAExportModel.java

@@ -1,5 +1,7 @@
 package eu.eudat.models.data.rda;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
 public class DatasetMetadataRDAExportModel {
 	private String description; // Not mandatory.
 	private String language;
@@ -29,7 +31,7 @@ public class DatasetMetadataRDAExportModel {
 	public DatasetMetadataRDAExportModel fromDataModel(String key, Object value) {
 		DatasetMetadataRDAExportModel metadataRDAExportModel = new DatasetMetadataRDAExportModel();
 		if (key.contains("metadata_standard_id"))
-			metadataRDAExportModel.setMetadata_standard_id(new IdRDAExportModel(value.toString(), value.toString()));
+			metadataRDAExportModel.setMetadata_standard_id(new IdRDAExportModel(value.toString(), "other"));
 		else if (key.contains("language"))
 			metadataRDAExportModel.setLanguage(value.toString());
 		else if (key.contains("description"))
@@ -37,4 +39,9 @@ public class DatasetMetadataRDAExportModel {
 
 		return metadataRDAExportModel;
 	}
+
+	@JsonIgnore
+	public boolean isValid() {
+		return description != null || language != null || metadata_standard_id != null;
+	}
 }

dmp-backend/web/src/main/java/eu/eudat/models/data/rda/DatasetRDAExportModel.java

@@ -1,12 +1,13 @@
 package eu.eudat.models.data.rda;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.jayway.jsonpath.JsonPath;
 import eu.eudat.data.entities.Dataset;
+import eu.eudat.logic.managers.DatasetManager;
 import eu.eudat.logic.utilities.builders.XmlBuilder;
-import org.apache.commons.collections4.MultiValuedMap;
-import org.apache.commons.collections4.multimap.ArrayListValuedHashMap;
-import org.apache.poi.ss.formula.functions.T;
+import org.json.JSONArray;
 import org.json.JSONObject;
 import org.w3c.dom.Document;
 import org.w3c.dom.Node;
@@ -16,12 +17,14 @@ import javax.xml.xpath.*;
 import java.text.DateFormat;
 import java.util.*;
 
+import static java.util.stream.Collectors.groupingBy;
+
 public class DatasetRDAExportModel {
 
 	private static final ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-	private MultiValuedMap<String, String> rdaToValueMap;
+	private Map<String, String> multiplicityIdToFieldSetId = new HashMap<>();
 
-	private String data_quality_assurance;
+	private List<String> data_quality_assurance;
 	private IdRDAExportModel dataset_id;
 	private String description;
 	private List<DatasetDistributionRDAExportModel> distribution;
@@ -37,10 +40,10 @@ public class DatasetRDAExportModel {
 	private String title;
 	private String type; // Type according to: http://vocabularies.coar-repositories.org/pubby/resource_type.html
 
-	public String getData_quality_assurance() {
+	public List<String> getData_quality_assurance() {
 		return data_quality_assurance;
 	}
-	public void setData_quality_assurance(String data_quality_assurance) {
+	public void setData_quality_assurance(List<String> data_quality_assurance) {
 		this.data_quality_assurance = data_quality_assurance;
 	}
 
@@ -143,28 +146,280 @@ public class DatasetRDAExportModel {
 	}
 
 
-	public DatasetRDAExportModel fromDataModel(Dataset dataset) {
-		this.rdaToValueMap = getRdaValueMap(dataset);
-		List<DatasetDistributionRDAExportModel> distributions = getDatasetDistribution();
-		List<DatasetMetadataRDAExportModel> datasetMetadata = getDatasetMetadata();
-		List<DatasetSecurityAndPrivacyRDAExportModel> securityAndPrivacy = getSecurityAndPrivacy();
-		List<DatasetTechnicalResourceRDAExportModel> technicalResources = getTechnicalResource();
+	public DatasetRDAExportModel fromDataModel(Dataset dataset, DatasetManager datasetManager) {
+		// Map of template Ids to rda values.
+		JSONObject jObject = new JSONObject(dataset.getProperties());
+		Map<String, Object> templateIdsToValues = jObject.toMap();
 
-		DatasetRDAExportModel datasetRDAExportModel = mapper.convertValue(this.rdaToValueMap, DatasetRDAExportModel.class);
+		/*--------- Building dataset rda export model ---------*/
+		DatasetRDAExportModel datasetRDAExportModel = new DatasetRDAExportModel();
 		datasetRDAExportModel.setDataset_id(new IdRDAExportModel(dataset.getId().toString(), "other"));
-		datasetRDAExportModel.setDescription(dataset.getDescription());
+		if (dataset.getDescription() != null) datasetRDAExportModel.setDescription(dataset.getDescription().replace("\n", " "));
 		datasetRDAExportModel.setIssued(DateFormat.getDateInstance(DateFormat.SHORT).format(dataset.getCreated()));
-		datasetRDAExportModel.setLanguage("en"); // mock data;
+		datasetRDAExportModel.setLanguage("en"); // mock data
 		datasetRDAExportModel.setTitle(dataset.getLabel());
-		datasetRDAExportModel.setDistribution(distributions);
-		datasetRDAExportModel.setMetadata(datasetMetadata);
-		datasetRDAExportModel.setSecurity_and_privacy(securityAndPrivacy);
-		datasetRDAExportModel.setTechnical_resource(technicalResources);
+
+		// Transform the answered dataset description to json so we can parse it and fill the rda model.
+		JSONObject datasetDescriptionJson = null;
+		try {
+			String jsonResult = mapper.writeValueAsString(datasetManager.getSingle(dataset.getId().toString()).getDatasetProfileDefinition());
+			datasetDescriptionJson = new JSONObject(jsonResult);
+		} catch (JsonProcessingException e) {
+			e.printStackTrace();
+		}
+		setMultiplicityIdToFieldSetId(datasetDescriptionJson);
+
+		/*--------- Building personal data. ---------*/
+		String personalData = buildSingleProperties("dataset.personal_data", datasetDescriptionJson, templateIdsToValues);
+		if (personalData != null) {
+			datasetRDAExportModel.setPersonal_data(personalData);
+		} else {
+			datasetRDAExportModel.setPersonal_data("unknown");
+		}
+
+		/*--------- Building preservation statement. ---------*/
+		datasetRDAExportModel.setPreservation_statement(buildSingleProperties("dataset.preservation_statement", datasetDescriptionJson, templateIdsToValues));
+
+		/*--------- Building sensitive data. ---------*/
+		String sensitiveData = buildSingleProperties("dataset.sensitive_data", datasetDescriptionJson, templateIdsToValues);
+		if (personalData != null) {
+			datasetRDAExportModel.setSensitive_data(sensitiveData);
+		} else {
+			datasetRDAExportModel.setSensitive_data("unknown");
+		}
+
+		/*--------- Building type. ---------*/
+		datasetRDAExportModel.setType(buildSingleProperties("dataset.type", datasetDescriptionJson, templateIdsToValues));
+
+		/*--------- Building data_quality_assurance. ---------*/
+		datasetRDAExportModel.setData_quality_assurance(buildDataQualityAssurance(datasetDescriptionJson, templateIdsToValues, dataset.getProfile().getDefinition()));
+
+		/*--------- Building keywords. ---------*/
+		datasetRDAExportModel.setKeyword(buildKeywords(datasetDescriptionJson, templateIdsToValues, dataset.getProfile().getDefinition()));
+
+		/*--------- Building metadata items. ---------*/
+		datasetRDAExportModel.setMetadata(buildMetadata(datasetDescriptionJson, templateIdsToValues, dataset.getProfile().getDefinition()));
+
+		/*--------- Building security and privacy items. ---------*/
+		datasetRDAExportModel.setSecurity_and_privacy(buildSecurityAndPrivacy(datasetDescriptionJson, templateIdsToValues, dataset.getProfile().getDefinition()));
+
+		/*--------- Building technical_resource. ---------*/
+		datasetRDAExportModel.setTechnical_resource(buildTechnicalResource(datasetDescriptionJson, templateIdsToValues, dataset.getProfile().getDefinition()));
 
 		return datasetRDAExportModel;
 	}
 
-	private List<String> xmlNodeListFromExpression(String xml, String expression) {
+	private String buildSingleProperties(String rdaKey, JSONObject datasetDescriptionJson, Map<String, Object> templateIdsToValues) {
+		String expression = "$..fields[*][?(@.rdaProperty == \"" + rdaKey + "\" )].id";
+		List<String> list = jsonValueListFromExpression(datasetDescriptionJson, expression);
+		if (!list.isEmpty()) {
+			return templateIdsToValues.get(list.get(0)).toString();
+		} else {
+			return null;
+		}
+	}
+
+	private List<String> buildDataQualityAssurance(JSONObject datasetDescriptionJson, Map<String, Object> templateIdsToValues, String datasetProfileDefinition) {
+		List<RdaField> dataQualityFields = getRDAFieldsFromJson(datasetDescriptionJson, new String[]{"dataset.data_quality_assurance"}, datasetProfileDefinition);
+		for (RdaField rdaField : dataQualityFields) {
+			rdaField.setRdaValue(templateIdsToValues.get(rdaField.getFieldId()).toString());
+		}
+		List<String> dataQualityAssuranceList = new LinkedList<>();
+		for (RdaField rdaField : dataQualityFields) {
+			dataQualityAssuranceList.add(rdaField.getRdaValue());
+		}
+
+		return dataQualityAssuranceList;
+	}
+
+	private List<String> buildKeywords(JSONObject datasetDescriptionJson, Map<String, Object> templateIdsToValues, String datasetProfileDefinition) {
+		List<RdaField> keywordFields = getRDAFieldsFromJson(datasetDescriptionJson, new String[]{"dataset.keyword"}, datasetProfileDefinition);
+		for (RdaField rdaField : keywordFields) {
+			rdaField.setRdaValue(templateIdsToValues.get(rdaField.getFieldId()).toString());
+		}
+		List<String> keywordsList = new LinkedList<>();
+		for (RdaField rdaField : keywordFields) {
+			keywordsList.add(rdaField.getRdaValue());
+		}
+
+		return keywordsList;
+	}
+
+	private List<DatasetMetadataRDAExportModel> buildMetadata(JSONObject datasetDescriptionJson, Map<String, Object> templateIdsToValues, String datasetProfileDefinition) {
+		List<RdaField> metadataFields = getRDAFieldsFromJson(datasetDescriptionJson,
+				new String[]{"dataset.metadata.metadata_standard_id.type", "dataset.metadata.metadata_standard_id.identifier", "dataset.metadata.description", "dataset.metadata.language", "dataset.metadata.metadata_standard_id"},
+				datasetProfileDefinition);
+
+		// Adding rdaValue and FieldSetIds on metadataFields.
+		for (RdaField rdaField : metadataFields) {
+			rdaField.setRdaValue(templateIdsToValues.get(rdaField.getFieldId()).toString());
+		}
+		// Group metadataFields based on their field set id.
+		Map<String, List<RdaField>> groupedMetadataFields = metadataFields.stream().collect(groupingBy(RdaField::getFieldSetId));
+
+		// Creating the metadata.
+		List<DatasetMetadataRDAExportModel> metadataRDAExportModelList = new LinkedList<>();
+		for (String fieldSetId : groupedMetadataFields.keySet()) {
+			DatasetMetadataRDAExportModel metadataRda = new DatasetMetadataRDAExportModel();
+			for (RdaField rdaField : groupedMetadataFields.get(fieldSetId)) {
+				if (rdaField.getRdaProperty().equals("dataset.metadata.metadata_standard_id.identifier")) {
+					if (metadataRda.getMetadata_standard_id() != null) {
+						metadataRda.getMetadata_standard_id().setIdentifier(rdaField.getRdaValue());
+					} else {
+						metadataRda.setMetadata_standard_id(new IdRDAExportModel(rdaField.getRdaValue(), "other"));
+					}
+				}
+				if (rdaField.getRdaProperty().equals("dataset.metadata.metadata_standard_id.type")) {
+					if (metadataRda.getMetadata_standard_id() != null) {
+						metadataRda.getMetadata_standard_id().setType(rdaField.getRdaValue());
+					} else {
+						metadataRda.setMetadata_standard_id(new IdRDAExportModel("", rdaField.getRdaValue()));
+					}
+				}
+				if (rdaField.getRdaProperty().equals("dataset.metadata.description")) {
+					metadataRda.setDescription(rdaField.getRdaValue());
+				}
+				if (rdaField.getRdaProperty().equals("dataset.metadata.language")) {
+					metadataRda.setLanguage(rdaField.getRdaValue());
+				}
+				if (rdaField.getRdaProperty().equals("dataset.metadata.metadata_standard_id")) {
+					JSONArray jsonArray = new JSONArray(rdaField.getRdaValue());
+					for (int i = 0; i < jsonArray.length(); i++) {
+						JSONObject jsonObject = jsonArray.getJSONObject(i);
+						Map<String, Object> jsonObjectMap = jsonObject.toMap();
+						DatasetMetadataRDAExportModel metadataRda1 = new DatasetMetadataRDAExportModel();
+						metadataRda1.setMetadata_standard_id(new IdRDAExportModel(jsonObjectMap.get("label").toString(), jsonObjectMap.get("source").toString()));
+						metadataRDAExportModelList.add(metadataRda1);
+					}
+				}
+			}
+			if (metadataRda.isValid()) {
+				metadataRDAExportModelList.add(metadataRda);
+			}
+		}
+
+		return new LinkedList<>(metadataRDAExportModelList);
+	}
+
+	private List<DatasetSecurityAndPrivacyRDAExportModel> buildSecurityAndPrivacy(JSONObject datasetDescriptionJson, Map<String, Object> templateIdsToValues, String datasetProfileDefinition) {
+		List<RdaField> secAndPrFields = getRDAFieldsFromJson(
+				datasetDescriptionJson,
+				new String[]{"dataset.security_and_privacy.description", "dataset.security_and_privacy.title", "dataset.security_and_privacy"},
+				datasetProfileDefinition);
+		for (RdaField rdaField : secAndPrFields) {
+			rdaField.setRdaValue(templateIdsToValues.get(rdaField.getFieldId()).toString());
+		}
+		Map<String, List<RdaField>> groupedSecurityAndPrivacyFields = secAndPrFields.stream().collect(groupingBy(RdaField::getFieldSetId));
+
+		List<DatasetSecurityAndPrivacyRDAExportModel> securityAndPrivacyRDAExportModelList = new LinkedList<>();
+		for (String fieldSetId : groupedSecurityAndPrivacyFields.keySet()) {
+			DatasetSecurityAndPrivacyRDAExportModel securityAndPrivacyModel = new DatasetSecurityAndPrivacyRDAExportModel();
+			for (RdaField rdaField : groupedSecurityAndPrivacyFields.get(fieldSetId)) {
+				if (rdaField.getRdaProperty().equals("dataset.security_and_privacy.description")) {
+					securityAndPrivacyModel.setDescription(rdaField.getRdaValue());
+				}
+				if (rdaField.getRdaProperty().equals("dataset.security_and_privacy.title")) {
+					securityAndPrivacyModel.setTitle(rdaField.getRdaValue());
+				}
+				if (rdaField.getRdaProperty().equals("dataset.security_and_privacy")) {
+					JSONArray jsonArray = new JSONArray(rdaField.getRdaValue());
+					for (int i = 0; i < jsonArray.length(); i++) {
+						JSONObject jsonObject = jsonArray.getJSONObject(i);
+						Map<String, Object> jsonObjectMap = jsonObject.toMap();
+						DatasetSecurityAndPrivacyRDAExportModel secAndPrivacy = new DatasetSecurityAndPrivacyRDAExportModel(jsonObjectMap.get("label").toString(), jsonObjectMap.get("source").toString());
+						securityAndPrivacyRDAExportModelList.add(secAndPrivacy);
+					}
+				}
+			}
+			securityAndPrivacyRDAExportModelList.add(securityAndPrivacyModel);
+		}
+
+		return securityAndPrivacyRDAExportModelList;
+	}
+
+	private List<DatasetTechnicalResourceRDAExportModel> buildTechnicalResource(JSONObject datasetDescriptionJson, Map<String, Object> templateIdsToValues, String datasetProfileDefinition) {
+		List<RdaField> dataQualityFields = getRDAFieldsFromJson(datasetDescriptionJson,
+				new String[]{"dataset.technical_resource.technical_resource", "dataset.technical_resource.technical_resource.description", "dataset.technical_resource.technical_resource.name"},
+				datasetProfileDefinition);
+		for (RdaField rdaField : dataQualityFields) {
+			rdaField.setRdaValue(templateIdsToValues.get(rdaField.getFieldId()).toString());
+		}
+		List<DatasetTechnicalResourceRDAExportModel> technicalResourceList = new LinkedList<>();
+		Map<String, List<RdaField>> groupedDataQualityFields = dataQualityFields.stream().collect(groupingBy(RdaField::getFieldSetId));
+		for (String fieldSetId : groupedDataQualityFields.keySet()) {
+			DatasetTechnicalResourceRDAExportModel technicalResourceModel = new DatasetTechnicalResourceRDAExportModel();
+			for (RdaField rdaField : groupedDataQualityFields.get(fieldSetId)) {
+				if (rdaField.getRdaProperty().equals("dataset.technical_resource.technical_resource.description")) {
+					technicalResourceModel.setDescription(rdaField.getRdaValue());
+				}
+				if (rdaField.getRdaProperty().equals("dataset.technical_resource.technical_resource.name")) {
+					technicalResourceModel.setName(rdaField.getRdaValue());
+				}
+				if (rdaField.getRdaProperty().equals("dataset.security_and_privacy")) {
+					JSONArray jsonArray = new JSONArray(rdaField.getRdaValue());
+					for (int i = 0; i < jsonArray.length(); i++) {
+						JSONObject jsonObject = jsonArray.getJSONObject(i);
+						Map<String, Object> jsonObjectMap = jsonObject.toMap();
+						DatasetTechnicalResourceRDAExportModel technicalResource = new DatasetTechnicalResourceRDAExportModel(jsonObjectMap.get("label").toString(), jsonObjectMap.get("label").toString());
+						technicalResourceList.add(technicalResource);
+					}
+				}
+			}
+			technicalResourceList.add(technicalResourceModel);
+		}
+
+		return technicalResourceList;
+	}
+
+	private void setMultiplicityIdToFieldSetId(JSONObject json) {
+		String multiplicityItemsFieldSetIdExp = "$..multiplicityItems[*].id";
+		List<String> multiplicityItemsFieldSetIdList = jsonValueListFromExpression(json, multiplicityItemsFieldSetIdExp);
+		for (String fieldSetId : multiplicityItemsFieldSetIdList) {
+			String fieldsFromFieldSetIdExp = "$..multiplicityItems[*][?(@.id == \""+ fieldSetId +"\")].fields[*].id";
+			List<String> fieldsIdList = jsonValueListFromExpression(json, fieldsFromFieldSetIdExp);
+			for (String fieldId : fieldsIdList) {
+				this.multiplicityIdToFieldSetId.put(fieldId, fieldSetId);
+			}
+		}
+	}
+
+	private List<RdaField> getRDAFieldsFromJson(JSONObject json, String[] rdaKey, String datasetProfileDefinition) {
+		List<RdaField> rdaFields = new LinkedList<>();
+		for (String key : rdaKey) {
+			String fieldIdExpression = "$..fields[*][?(@.rdaProperty == \"" + key + "\" )].id";
+			List<String> listFromExpression = jsonValueListFromExpression(json, fieldIdExpression);
+			for (String fieldId : listFromExpression) {
+				RdaField rdaField = new RdaField();
+				rdaField.setRdaProperty(key);
+				rdaField.setFieldId(fieldId);
+				if (fieldId.startsWith("multiple_")) {
+					rdaField.setFieldSetId(this.multiplicityIdToFieldSetId.get(fieldId));
+				} else {
+					rdaField.setFieldSetId(getFieldSetIdForFieldFromXML(datasetProfileDefinition, fieldId));
+				}
+				rdaFields.add(rdaField);
+			}
+		}
+		return rdaFields;
+	}
+
+	private List<String> jsonValueListFromExpression(JSONObject json, String expression) {
+		net.minidev.json.JSONArray jsonArray = JsonPath.parse(json.toString()).read(expression);
+		List<String> valueList = new LinkedList<>();
+		for (Object o : jsonArray) {
+			valueList.add(o.toString());
+		}
+		return valueList;
+	}
+
+	private String getFieldSetIdForFieldFromXML(String datasetProfileDefinition, String fieldId) {
+		String fieldSetIdExpression = "//field[@id ='" + fieldId + "']/ancestor::fieldSet/@id";
+		List<String> listFromExpression = xmlValueListFromExpression(datasetProfileDefinition, fieldSetIdExpression);
+		if (listFromExpression.size() == 1) return listFromExpression.get(0);
+		return null;
+	}
+
+	private List<String> xmlValueListFromExpression(String xml, String expression) {
 		List<String> valuesList = new LinkedList<>();
 		Document document = XmlBuilder.fromXml(xml);
 		XPathFactory xpathFactory = XPathFactory.newInstance();
@@ -182,117 +437,4 @@ public class DatasetRDAExportModel {
 
 		return valuesList;
 	}
-
-	private Map<String,String> combineListsIntoOrderedMap (List<String> keys, List<String> values) {
-		if (keys.size() != values.size())
-			throw new IllegalArgumentException ("Cannot combine lists with dissimilar sizes");
-		Map<String,String> map = new LinkedHashMap<>();
-		for (int i=0; i<keys.size(); i++) {
-			map.put(keys.get(i), values.get(i));
-		}
-		return map;
-	}
-
-	private MultiValuedMap<String, String> getRdaValueMap(Dataset dataset) {
-		// Parsing dataset template definition to create a map of which question of the template corresponds to the RDA common standard. Map: TemplateId -> rdaProperty
-		List<String> rdaPropertiesThatExistOnDatasetTemplate = xmlNodeListFromExpression(dataset.getProfile().getDefinition(), "//rdaCommonStandard/text()");
-		List<String> rdaProperties = new LinkedList<>();
-		for (String item : rdaPropertiesThatExistOnDatasetTemplate) {
-			item = item.replace("dataset.", "");
-			rdaProperties.add(item);
-		}
-		List<String> datasetTemplateIdsWithRdaProperties = xmlNodeListFromExpression(dataset.getProfile().getDefinition(), "//rdaCommonStandard/text()/ancestor::field/@id");
-		Map<String, String> templateIdsToRDAProperties = combineListsIntoOrderedMap(datasetTemplateIdsWithRdaProperties, rdaProperties);
-
-		// Parsing dataset answers from json to map. Map: TemplateId -> datasetValue
-		JSONObject jObject = new JSONObject(dataset.getProperties());
-		Map<String, Object> templateIdsToValues = jObject.toMap();
-
-		// Map: rdaProperty -> datasetValue
-		MultiValuedMap<String, String> rdaToValueMap = new ArrayListValuedHashMap<>();
-		for (String templateId : templateIdsToRDAProperties.keySet()) {
-			if (templateIdsToValues.containsKey(templateId)) {
-				rdaToValueMap.put(templateIdsToRDAProperties.get(templateId), templateIdsToValues.get(templateId).toString());
-			}
-		}
-		return rdaToValueMap;
-	}
-
-	private List<DatasetDistributionRDAExportModel> getDatasetDistribution() {
-		// Creates and fills a map to include distribution keys and values.
-		MultiValuedMap<String, String> distributionJsonMap = new ArrayListValuedHashMap<>();
-		for (String key : this.rdaToValueMap.keySet()) {
-			if (key.startsWith("distribution.")) {
-				distributionJsonMap.putAll(key.replace("distribution.", ""), this.rdaToValueMap.get(key));
-			}
-		}
-		// Removes distribution keys from rdaToValueMap.
-		for (String distributionKey : distributionJsonMap.keySet()) this.rdaToValueMap.remove(distributionKey);
-		List<DatasetDistributionRDAExportModel> distributionModels = new LinkedList<>();
-		for (String key : distributionJsonMap.keySet()) {
-			for (String value : distributionJsonMap.get(key)) {
-				distributionModels.add(new DatasetDistributionRDAExportModel().fromDataModel(key, value));
-			}
-		}
-		return distributionModels;
-	}
-
-	private List<DatasetMetadataRDAExportModel> getDatasetMetadata() {
-		// Creates and fills a map to include metadata keys and values.
-		MultiValuedMap<String, String> metadataJsonMap = new ArrayListValuedHashMap<>();
-		for (String key : this.rdaToValueMap.keySet()) {
-			if (key.startsWith("metadata.")) {
-				metadataJsonMap.putAll(key.replace("metadata.", ""), this.rdaToValueMap.get(key));
-			}
-		}
-		// Removes metadata keys from rdaToValueMap.
-		for (String metadataKey : metadataJsonMap.keySet()) this.rdaToValueMap.remove(metadataKey);
-		List<DatasetMetadataRDAExportModel> datasetMetadataRDAExportModels = new LinkedList<>();
-		for (String key : metadataJsonMap.keySet()) {
-			for (String value : metadataJsonMap.get(key)) {
-				datasetMetadataRDAExportModels.add(new DatasetMetadataRDAExportModel().fromDataModel(key, value));
-			}
-		}
-		return datasetMetadataRDAExportModels;
-	}
-
-	private List<DatasetSecurityAndPrivacyRDAExportModel> getSecurityAndPrivacy() {
-		// Creates and fills a map to include SecurityAndPrivacy keys and values.
-		MultiValuedMap<String, String> securityAndPrivacyJsonMap = new ArrayListValuedHashMap<>();
-		for (String key : this.rdaToValueMap.keySet()) {
-			if (key.startsWith("security_and_privacy.")) {
-				securityAndPrivacyJsonMap.putAll(key.replace("security_and_privacy.", ""), this.rdaToValueMap.get(key));
-			}
-		}
-		// Removes SecurityAndPrivacy keys from rdaToValueMap.
-		for (String securityAndPrivacy : securityAndPrivacyJsonMap.keySet()) this.rdaToValueMap.remove(securityAndPrivacy);
-		List<DatasetSecurityAndPrivacyRDAExportModel> securityAndPrivacyList= new LinkedList<>();
-		for (String key : securityAndPrivacyJsonMap.keySet()) {
-			for (String value : securityAndPrivacyJsonMap.get(key)) {
-				securityAndPrivacyList.add(new DatasetSecurityAndPrivacyRDAExportModel().fromDataModel(key, value));
-			}
-		}
-		return securityAndPrivacyList;
-	}
-
-	private List<DatasetTechnicalResourceRDAExportModel> getTechnicalResource() {
-		// Creates and fills a map to include Technical_Resource keys and values.
-		MultiValuedMap<String, String> technicalResourceJsonMap = new ArrayListValuedHashMap<>();
-		for (String key : this.rdaToValueMap.keySet()) {
-			if (key.startsWith("technical_resource.")) {
-				technicalResourceJsonMap.putAll(key.replace("technical_resource.", ""), this.rdaToValueMap.get(key));
-			}
-		}
-		// Removes Technical_Resource keys from rdaToValueMap.
-		for (String techResource : technicalResourceJsonMap.keySet()) this.rdaToValueMap.remove(techResource);
-		List<DatasetTechnicalResourceRDAExportModel> datasetTechnicalResourceRDAExportModel= new LinkedList<>();
-		for (String key : technicalResourceJsonMap.keySet()) {
-			for (String value : technicalResourceJsonMap.get(key)) {
-				datasetTechnicalResourceRDAExportModel.add(new DatasetTechnicalResourceRDAExportModel().fromDataModel(key, value));
-			}
-		}
-		return datasetTechnicalResourceRDAExportModel;
-	}
-
-
 }

dmp-backend/web/src/main/java/eu/eudat/models/data/rda/DatasetSecurityAndPrivacyRDAExportModel.java

@@ -4,6 +4,15 @@ public class DatasetSecurityAndPrivacyRDAExportModel {
 	private String description;
 	private String title;
 
+	public DatasetSecurityAndPrivacyRDAExportModel() {
+
+	}
+
+	public DatasetSecurityAndPrivacyRDAExportModel(String description, String title) {
+		this.description = description;
+		this.title = title;
+	}
+
 	public String getDescription() {
 		return description;
 	}
@@ -17,13 +26,4 @@ public class DatasetSecurityAndPrivacyRDAExportModel {
 	public void setTitle(String title) {
 		this.title = title;
 	}
-
-	public DatasetSecurityAndPrivacyRDAExportModel fromDataModel(String key, Object value) {
-		DatasetSecurityAndPrivacyRDAExportModel securityAndPrivacy = new DatasetSecurityAndPrivacyRDAExportModel();
-		if (key.contains("description"))
-			securityAndPrivacy.setDescription(value.toString());
-		else if (key.contains("title"))
-			securityAndPrivacy.setTitle(value.toString());
-		return securityAndPrivacy;
-	}
 }

dmp-backend/web/src/main/java/eu/eudat/models/data/rda/DatasetTechnicalResourceRDAExportModel.java

@@ -18,12 +18,11 @@ public class DatasetTechnicalResourceRDAExportModel {
 		this.name = name;
 	}
 
-	public DatasetTechnicalResourceRDAExportModel fromDataModel(String key, Object value) {
-		DatasetTechnicalResourceRDAExportModel technicalResource = new DatasetTechnicalResourceRDAExportModel();
-		if (key.contains("description"))
-			technicalResource.setDescription(value.toString());
-		else if (key.contains("name"))
-			technicalResource.setName(value.toString());
-		return technicalResource;
+	public DatasetTechnicalResourceRDAExportModel(String description, String name) {
+		this.description = description;
+		this.name = name;
 	}
-}
+
+	public DatasetTechnicalResourceRDAExportModel() {
+	}
+}

dmp-backend/web/src/main/java/eu/eudat/models/data/rda/DmpRDAExportModel.java

@@ -4,6 +4,7 @@ import eu.eudat.data.entities.DMP;
 import eu.eudat.data.entities.Dataset;
 import eu.eudat.data.entities.UserDMP;
 import eu.eudat.data.entities.UserInfo;
+import eu.eudat.logic.managers.DatasetManager;
 
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
@@ -126,7 +127,7 @@ public class DmpRDAExportModel {
         this.title = title;
     }
 
-    public DmpRDAExportModel fromDataModel(DMP entity) {
+    public DmpRDAExportModel fromDataModel(DMP entity, DatasetManager datasetManager) {
         DmpRDAExportModel dmpRda = new DmpRDAExportModel();
         dmpRda.contact = new ContactRDAExportModel().fromDataModel(entity.getUsers().stream().filter(x -> x.getRole().equals(UserDMP.UserDMPRoles.OWNER.getValue())).findFirst().get().getUser());
     	if (entity.getUsers().stream().anyMatch(x -> x.getRole().equals(UserDMP.UserDMPRoles.USER.getValue()))) {
@@ -141,9 +142,9 @@ public class DmpRDAExportModel {
         dmpRda.dataset = new LinkedList<>();
         for (Dataset dataset : entity.getDataset()) {
             if (dataset.getStatus() != Dataset.Status.DELETED.getValue() && dataset.getStatus() != Dataset.Status.CANCELED.getValue())
-                dmpRda.dataset.add(new DatasetRDAExportModel().fromDataModel(dataset));
+                dmpRda.dataset.add(new DatasetRDAExportModel().fromDataModel(dataset, datasetManager));
         }
-        dmpRda.description = entity.getDescription();
+        dmpRda.description = entity.getDescription().replace("\n", " ");
         if (entity.getDoi() != null) {
             dmpRda.dmp_id = new IdRDAExportModel(entity.getDoi(), "zenodo");
         }
@@ -151,9 +152,7 @@ public class DmpRDAExportModel {
             dmpRda.dmp_id = new IdRDAExportModel(entity.getId().toString(), "other");
         }
         // Mock up data on "language" and "ethical_issues_*" for now.
-        // dmpRda.ethical_issues_description = null;
         dmpRda.ethical_issues_exist = "unknown";
-        // dmpRda.ethical_issues_report = null;
         dmpRda.language = "en";
         dmpRda.modified = formatter.format(new Date());
         dmpRda.project = new ProjectRDAExportModel().fromDataModel(entity.getGrant());

dmp-backend/web/src/main/java/eu/eudat/models/data/rda/RDAExportModel.java

@@ -2,6 +2,7 @@ package eu.eudat.models.data.rda;
 
 import eu.eudat.data.entities.DMP;
 import eu.eudat.data.entities.Dataset;
+import eu.eudat.logic.managers.DatasetManager;
 
 import java.util.LinkedList;
 import java.util.List;
@@ -16,8 +17,8 @@ public class RDAExportModel {
         this.dmp = dmp;
     }
 
-    public RDAExportModel fromDataModel(DMP dmp) {
-        this.dmp = new DmpRDAExportModel().fromDataModel(dmp);
+    public RDAExportModel fromDataModel(DMP dmp, DatasetManager datasetManager) {
+        this.dmp = new DmpRDAExportModel().fromDataModel(dmp, datasetManager);
         return this;
     }
 }

dmp-backend/web/src/main/java/eu/eudat/models/data/rda/RdaField.java

@@ -0,0 +1,36 @@
+package eu.eudat.models.data.rda;
+
+public class RdaField {
+	private String rdaProperty;
+	private String rdaValue;
+	private String fieldId;
+	private String fieldSetId;
+
+	public String getRdaProperty() {
+		return rdaProperty;
+	}
+	public void setRdaProperty(String rdaProperty) {
+		this.rdaProperty = rdaProperty;
+	}
+
+	public String getRdaValue() {
+		return rdaValue;
+	}
+	public void setRdaValue(String rdaValue) {
+		this.rdaValue = rdaValue;
+	}
+
+	public String getFieldId() {
+		return fieldId;
+	}
+	public void setFieldId(String fieldId) {
+		this.fieldId = fieldId;
+	}
+
+	public String getFieldSetId() {
+		return fieldSetId;
+	}
+	public void setFieldSetId(String fieldSetId) {
+		this.fieldSetId = fieldSetId;
+	}
+}

dmp-backend/web/src/main/java/eu/eudat/models/data/user/components/datasetprofile/Field.java

@@ -28,6 +28,7 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
     private List<Field> multiplicityItems;
     private List<eu.eudat.models.data.admin.components.datasetprofile.Field.ValidationType> validations;
     private Visibility visible;
+    private String rdaProperty;
 
     public List<Field> getMultiplicityItems() {
         return multiplicityItems;
@@ -137,6 +138,14 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
         this.numbering = numbering;
     }
 
+    public String getRdaProperty() {
+        return rdaProperty;
+    }
+
+    public void setRdaProperty(String rdaProperty) {
+        this.rdaProperty = rdaProperty;
+    }
+
     public Field cloneForMultiplicity(String key, Map<String, Object> properties) {
         Field newField = new Field();
         newField.id = key;
@@ -148,6 +157,7 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
         newField.defaultValue = this.defaultValue;
         newField.data = this.data;
         newField.validations = this.validations;
+        newField.rdaProperty = this.rdaProperty;
         return newField;
     }
 
@@ -160,6 +170,7 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
         field.setDefaultValue(this.defaultValue);
         field.setVisible(this.visible);
         field.setValidations(this.validations);
+        field.setRdaCommonStandard(this.rdaProperty);
         return field;
     }
 
@@ -173,6 +184,7 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
         this.defaultValue = item.getDefaultValue();
         this.visible = item.getVisible();
         this.validations = item.getValidations();
+        this.rdaProperty = item.getRdaCommonStandard();
     }
 
     @Override

dmp-backend/web/src/main/resources/RDACommonStandards.txt

@@ -22,6 +22,7 @@ dataset.distribution.license.start_date
 dataset.distribution.title
 dataset.keyword
 dataset.language
+dataset.metadata.description
 dataset.metadata.language
 dataset.metadata.metadata_standard_id
 dataset.metadata.metadata_standard_id.identifier