Fixing group and tag generation on Item Validator

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/gcat@177325 82a268e6-3cf1-43bd-a215-b396298e98cf

distro/changelog.xml

@@ -2,7 +2,9 @@
 <!DOCTYPE xml>
 <ReleaseNotes>
 	<Changeset component="org.gcube.data-publishing.gcat.1-2-0" date="${buildDate}">
-		<Change>Saparated REST class for Profile management from the logic which effectively manage profile on IS</Change>
+		<Change>Separated REST class for Profile management from the logic which effectively manage profile on IS</Change>
+		<Change>Tags are now properly created/added according to profile definition #16182</Change>
+		<Change>Groups are now properly created/added according to profile definition #16183</Change>
 	</Changeset>
 	<Changeset component="org.gcube.data-publishing.gcat.1-1-0" date="2019-02-26">
 		<Change>Added Item URL via URI Resolver in extras field #13309</Change>

src/main/java/org/gcube/gcat/oldutils/Validator.java

@@ -12,6 +12,9 @@ import java.util.Map.Entry;
 import java.util.Set;
 
 import javax.ws.rs.BadRequestException;
+import javax.ws.rs.InternalServerErrorException;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Response.Status;
 
 import org.apache.commons.lang.math.NumberUtils;
 import org.gcube.common.scope.api.ScopeProvider;
@@ -23,7 +26,9 @@ import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.MetadataGrouping;
 import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.MetadataTagging;
 import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.NamespaceCategory;
 import org.gcube.gcat.persistence.ckan.CKAN;
+import org.gcube.gcat.persistence.ckan.CKANGroup;
 import org.gcube.gcat.persistence.ckan.CKANPackage;
+import org.gcube.gcat.persistence.ckan.CKANUser;
 import org.gcube.gcat.persistence.ckan.CKANUtility;
 import org.gcube.gcat.profile.MetadataUtility;
 import org.gcube.gcat.social.SocialService;
@@ -50,29 +55,31 @@ public class Validator {
 	private static final SimpleDateFormat DATE_SIMPLE = new SimpleDateFormat("yyyy-MM-dd");
 	private static final SimpleDateFormat DATE_HOUR_MINUTES = new SimpleDateFormat("yyyy-MM-dd HH:mm");
 	
-	/**
+	public static final int MAX_TAG_CHARS = 100;
-	 * This method validate the incoming json dataset wrt a metadata profile
+	
-	 * @param json
+	protected ObjectMapper mapper;
-	 * @param caller
+	
-	 * @param profiles 
+	public Validator() {
-	 * @return
+		this.mapper = new ObjectMapper();
-	 * @throws Exception 
+	}
-	 */
+	
-	public static ObjectNode validateAgainstProfile(ObjectMapper mapper, ObjectNode objectNode) throws Exception {
+	public Validator(ObjectMapper mapper) {
-		
+		this.mapper = mapper;
+	}
+	
+	public ObjectNode validateAgainstProfile(ObjectNode objectNode) throws Exception {
 		
 		ArrayNode extrasArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.EXTRAS_KEY);
-		ArrayNode groupsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.GROUPS_KEY);
-		ArrayNode tagsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.TAGS_KEY);
-		
 		if(extrasArrayOriginal == null || extrasArrayOriginal.size()==0) {
 			throw new BadRequestException("'extras' field is missing in context where metadata profile(s) are defined!");
 		}
 		
+		ArrayNode groupsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.GROUPS_KEY);
 		if(groupsArrayOriginal == null) {
 			groupsArrayOriginal = mapper.createArrayNode();
 		}
 		
+		ArrayNode tagsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.TAGS_KEY);
 		if(tagsArrayOriginal == null) {
 			tagsArrayOriginal = mapper.createArrayNode();
 		}
@@ -147,7 +154,7 @@ public class Validator {
 					List<CustomField> validCFs = validateAgainstMetadataField(metadataIndex, categoryIdIndex,
 							customFields, tagsArrayOriginal, groupsArrayOriginal, metadataField, categories,
 							fieldsMandatoryLowerBoundMap, fieldsMandatoryUpperBoundMap, numberFieldsMandatorySameKeyMap,
-							groupsToCreateAfterValidation, mapper);
+							groupsToCreateAfterValidation);
 					validatedCustomFields.addAll(validCFs);
 					metadataIndex++;
 					
@@ -204,9 +211,9 @@ public class Validator {
 				// create groups
 				for(String title : groupsToCreateAfterValidation) {
 					try {
-						createGroupAsSysAdmin(title, title, "");
+						createGroupAsSysAdmin(title);
 					} catch(Exception e) {
-						logger.error("Failed to create group with title " + title, e);
+						logger.trace("Failed to create group with title " + title, e);
 					}
 				}
 			}
@@ -226,10 +233,47 @@ public class Validator {
 	 * @return
 	 * @throws Exception 
 	 */
-	public static CkanGroup createGroupAsSysAdmin(String title, String groupName, String description) throws Exception {
+	public void createGroupAsSysAdmin(String groupName) throws Exception {
-		return CKAN.getCatalogue().createGroup(groupName, title, description);
+		String sysAdminAPI = CKANUtility.getSysAdminAPI();
+		CKANGroup ckanGroup = new CKANGroup();
+		ckanGroup.setApiKey(sysAdminAPI);
+		ckanGroup.setName(CKANGroup.getGroupName(groupName));
+		try {
+			ckanGroup.read();
+		} catch (WebApplicationException e) {
+			if(e.getResponse().getStatus()==Status.NOT_FOUND.getStatusCode()) {
+				ckanGroup.create();
+			}else {
+				throw e;
+			}
+		}catch (Exception e) {
+			throw new InternalServerErrorException(e);
+		}finally {
+			try {
+				addUserToGroupAsSysAdmin(groupName);
+			}catch (WebApplicationException e) {
+				throw e;
+			}catch (Exception e) {
+				throw new InternalServerErrorException(e);
+			}
+		}
 	}
 	
+	public void addUserToGroupAsSysAdmin(String groupName) throws Exception {
+		String username = CKANUtility.getCKANUsername();
+		addUserToGroupAsSysAdmin(groupName, username);
+	}
+	
+	
+	public void addUserToGroupAsSysAdmin(String groupName, String username) throws Exception {
+		String sysAdminAPI = CKANUtility.getSysAdminAPI();
+		CKANUser ckanUser = new CKANUser();
+		ckanUser.setApiKey(sysAdminAPI);
+		ckanUser.setName(username);
+		ckanUser.addToGroup(CKANGroup.getGroupName(groupName));
+	}
+	
+	
 	/**
 	 * Validate this field and generate a new value (or returns the same if there is nothing to update)
 	 * @param metadataIndex
@@ -245,11 +289,11 @@ public class Validator {
 	 * @return
 	 * @throws Exception
 	 */
-	private static List<CustomField> validateAgainstMetadataField(int metadataIndex, int categoryIndex,
+	private List<CustomField> validateAgainstMetadataField(int metadataIndex, int categoryIndex,
 			List<CustomField> customFields, ArrayNode tagsArrayOriginal, ArrayNode groupsArrayOriginal,
 			MetadataField metadataField, List<NamespaceCategory> categories,
 			Map<String,Integer> fieldsMandatoryLowerBoundMap, Map<String,Integer> fieldsMandatoryUpperBoundMap,
-			Map<String,Integer> numberFieldsMandatorySameKeyMap, List<String> groupToCreate, ObjectMapper mapper) throws Exception {
+			Map<String,Integer> numberFieldsMandatorySameKeyMap, List<String> groupToCreate) throws Exception {
 		
 		List<CustomField> toReturn = new ArrayList<CustomField>();
 		String metadataFieldName = metadataField.getCategoryFieldQName(); // get the qualified one, if any
@@ -263,8 +307,8 @@ public class Validator {
 				fieldsFoundWithThisKey++;
 				cf.setIndexCategory(categoryIndex);
 				cf.setIndexMetadataField(metadataIndex);
-				checkAsGroup(cf, metadataField, groupsArrayOriginal, groupToCreate, mapper);
+				checkAsGroup(cf, metadataField, groupsArrayOriginal, groupToCreate);
-				checkAsTag(cf, metadataField, tagsArrayOriginal, mapper);
+				checkAsTag(cf, metadataField, tagsArrayOriginal);
 				toReturn.add(cf);
 				iterator.remove();
 			}
@@ -306,7 +350,7 @@ public class Validator {
 		
 	}
 	
-	public static final int MAX_TAG_CHARS = 100;
+	
 	
 	/**
 	 * Check if a tag must be generated
@@ -314,8 +358,8 @@ public class Validator {
 	 * @param metadataField
 	 * @param tagsArrayOriginal
 	 */
-	private static void checkAsTag(CustomField fieldToValidate, MetadataField metadataField,
+	private void checkAsTag(CustomField fieldToValidate, MetadataField metadataField,
-			ArrayNode tagsArrayOriginal, ObjectMapper mapper) {
+			ArrayNode tagsArrayOriginal) {
 		MetadataTagging tagging = metadataField.getTagging();
 		if(tagging != null) {
 			
@@ -358,8 +402,8 @@ public class Validator {
 	 * @param isApplication 
 	 * @throws Exception 
 	 */
-	private static void checkAsGroup(CustomField fieldToValidate, MetadataField metadataField,
+	private void checkAsGroup(CustomField fieldToValidate, MetadataField metadataField,
-			ArrayNode groupsArrayOriginal, List<String> groupToCreate, ObjectMapper mapper) throws Exception {
+			ArrayNode groupsArrayOriginal, List<String> groupToCreate) throws Exception {
 		
 		logger.debug("Custom field is " + fieldToValidate);
 		logger.debug("MetadataField field is " + metadataField);
@@ -422,7 +466,7 @@ public class Validator {
 	 * @return
 	 * @throws Exception 
 	 */
-	private static void validate(CustomField fieldToValidate, MetadataField metadataField) throws Exception {
+	private void validate(CustomField fieldToValidate, MetadataField metadataField) throws Exception {
 		
 		DataType dataType = metadataField.getDataType();
 		String regex = metadataField.getValidator() != null ? metadataField.getValidator().getRegularExpression()

src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java

@@ -268,6 +268,9 @@ public abstract class CKAN {
 		int responseCode  = httpURLConnection.getResponseCode();
 		if(responseCode >= Status.BAD_REQUEST.getStatusCode()) {
 			Status status = Status.fromStatusCode(responseCode);
+			InputStream inputStream = httpURLConnection.getErrorStream();
+			StringBuilder result = getStringBuilder(inputStream);
+			logger.trace(result.toString());
 			throw new WebApplicationException(status);
 		}
 		InputStream inputStream = httpURLConnection.getInputStream();

src/main/java/org/gcube/gcat/persistence/ckan/CKANGroup.java

@@ -1,5 +1,10 @@
 package org.gcube.gcat.persistence.ckan;
 
+import javax.ws.rs.InternalServerErrorException;
+import javax.ws.rs.WebApplicationException;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
 /**
  * @author Luca Frosini (ISTI - CNR)
  */
@@ -31,4 +36,22 @@ public class CKANGroup extends CKAN {
 		PURGE = GROUP_PURGE;
 	}
 	
+	public static String getGroupName(String name) {
+		return name.trim().toLowerCase().replaceAll(" ", "_");
+	}
+	
+	public String create() throws WebApplicationException {
+		try {
+			ObjectNode objectNode = mapper.createObjectNode();
+			objectNode.put(NAME_KEY, CKANGroup.getGroupName(name));
+			objectNode.put("title", name);
+			objectNode.put("display_name", name);
+			objectNode.put("description", "");
+			return super.create(mapper.writeValueAsString(objectNode));
+		}catch (WebApplicationException e) {
+			throw e;
+		}catch (Exception e) {
+			throw new InternalServerErrorException(e);
+		}
+	}
 }

src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java

@@ -200,7 +200,8 @@ public class CKANPackage extends CKAN {
 			// Validating against profiles if any
 			MetadataUtility metadataUtility = MetadataUtility.getInstance();
 			if(!metadataUtility.getMetadataProfiles().isEmpty()) {
-				objectNode = Validator.validateAgainstProfile(mapper, objectNode);
+				Validator validator = new Validator(mapper);
+				objectNode = validator.validateAgainstProfile(objectNode);
 			}
 			
 			return objectNode;

src/main/java/org/gcube/gcat/persistence/ckan/CKANUser.java

@@ -1,5 +1,8 @@
 package org.gcube.gcat.persistence.ckan;
 
+import javax.ws.rs.InternalServerErrorException;
+import javax.ws.rs.WebApplicationException;
+
 import org.gcube.gcat.utils.RandomString;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -21,6 +24,7 @@ public class CKANUser extends CKAN {
 	// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.delete.user_delete
 	public static final String USER_DELETE = CKAN.CKAN_API_PATH + "user_delete";
 	
+	public static final String ADD_USER_TO_GROUP = CKAN.CKAN_API_PATH + "member_create";
 	
 	public CKANUser() {
 		super();
@@ -51,4 +55,19 @@ public class CKANUser extends CKAN {
 		this.delete();
 	}
 	
+	
+	public void addToGroup(String groupName) throws WebApplicationException {
+		try {
+			ObjectNode objectNode = mapper.createObjectNode();
+			objectNode.put(ID_KEY, CKANGroup.getGroupName(groupName));
+			objectNode.put("object", name);
+			objectNode.put("object_type", "user");
+			objectNode.put("capacity", "member");
+			sendPostRequest(ADD_USER_TO_GROUP, getAsString(objectNode));
+		}catch (WebApplicationException e) {
+			throw e;
+		}catch (Exception e) {
+			throw new InternalServerErrorException(e);
+		}
+	}
 }

src/main/java/org/gcube/gcat/persistence/ckan/CKANUtility.java

@@ -45,7 +45,7 @@ public class CKANUtility {
 		return ckanUsername;
 	}
 	
-	protected static String getCKANUsername() {
+	public static String getCKANUsername() {
 		return getCKANUsername(ContextUtility.getUsername());
 	}
 	

src/test/java/org/gcube/gcat/oldutils/ValidatorTest.java

@@ -0,0 +1,14 @@
+package org.gcube.gcat.oldutils;
+
+import org.gcube.gcat.ContextTest;
+import org.junit.Test;
+
+public class ValidatorTest extends ContextTest {
+	
+	@Test
+	public void createGroupAsSysAdmin() throws Exception {
+		String groupName = "Italian";
+		Validator validator = new Validator();
+		validator.createGroupAsSysAdmin(groupName);
+	}
+}

src/test/java/org/gcube/gcat/persistence/ckan/CKANGroupTest.java

@@ -0,0 +1,30 @@
+package org.gcube.gcat.persistence.ckan;
+
+import org.gcube.gcat.ContextTest;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CKANGroupTest extends ContextTest {
+	
+	private static Logger logger = LoggerFactory.getLogger(CKANGroupTest.class);
+	
+	@Test
+	public void read() throws Exception {
+		CKANGroup ckanGroup = new CKANGroup();
+		ckanGroup.setApiKey(CKANUtility.getSysAdminAPI());
+		String name = "";
+		ckanGroup.setName(name);
+		String ret = ckanGroup.read();
+		logger.debug("{}", ret);
+	}
+	
+	// @Test
+	public void delete() throws Exception {
+		CKANGroup ckanGroup = new CKANGroup();
+		ckanGroup.setApiKey(CKANUtility.getSysAdminAPI());
+		String name = "";
+		ckanGroup.setName(name);
+		ckanGroup.delete(true);
+	}
+}

src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java

@@ -27,7 +27,7 @@ public class CKANPackageTest extends ContextTest {
 	
 	private static final String ITEM_NAME_VALUE = "restful_transaction_model";
 	private static final String LICENSE_VALUE = "CC-BY-SA-4.0";
-	private static final String EXTRAS_TYPE_VALUE_VALUE = "FSKXModel";
+	private static final String EXTRAS_TYPE_VALUE_VALUE = "TestProfile";
 	
 	
 	@Test
@@ -109,8 +109,8 @@ public class CKANPackageTest extends ContextTest {
 		extraArrayNode.add(typeNode);
 		
 		ObjectNode modelNode = mapper.createObjectNode();
-		modelNode.put(CKANPackage.EXTRAS_KEY_KEY, "Model Language");
+		modelNode.put(CKANPackage.EXTRAS_KEY_KEY, "test");
-		modelNode.put(CKANPackage.EXTRAS_VALUE_KEY, "Other");
+		modelNode.put(CKANPackage.EXTRAS_VALUE_KEY, "test");
 		extraArrayNode.add(modelNode);
 		
 		ObjectNode populationNode = mapper.createObjectNode();

src/test/java/org/gcube/gcat/profile/ProfileTest.java

@@ -57,7 +57,7 @@ public class ProfileTest extends ContextTest {
 	
 	@Test
 	public void testCreateOrUpdate() throws Exception {
-		String xml = "<metadataformat type=\"TestProfile\"><metadatafield><fieldName>test</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>1</maxOccurs><note>Test Field</note></metadatafield><metadatafield categoryref=\"FSKX_model_scope\"><fieldName>Population</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>*</maxOccurs><defaultValue/><note>The population of the model</note><tagging create=\"true\" separator=\" \">onValue</tagging><grouping create=\"true\">onFieldName_onValue</grouping></metadatafield></metadataformat>";
+		String xml = "<metadataformat type=\"TestProfile\"><metadatafield><fieldName>test</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>1</maxOccurs><note>Test Field</note></metadatafield><metadatafield><fieldName>Population</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>*</maxOccurs><defaultValue/><note>The population of the model</note><tagging create=\"true\" separator=\"_\">onValue</tagging><grouping create=\"true\">onFieldName_onValue</grouping></metadatafield></metadataformat>";
 		ISProfile profile = new ISProfile();
 		profile.createOrUpdate("TestProfile", xml);
 	}

src/test/java/org/gcube/gcat/rest/ProfileTest.java

@@ -1,92 +0,0 @@
-package org.gcube.gcat.rest;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.Iterator;
-import java.util.stream.Collectors;
-
-import javax.ws.rs.core.MediaType;
-
-import org.gcube.gcat.ContextTest;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-
-public class ProfileTest extends ContextTest {
-	
-	private static Logger logger = LoggerFactory.getLogger(ProfileTest.class);
-	
-	@Test
-	public void list() throws Exception {
-		Profile profile = new Profile();
-		String ret = profile.list();
-		logger.debug("{}", ret);
-	}
-	
-	
-	@Test
-	public void read() throws Exception {
-		String profileID = "EmptyProfile";
-		Profile profile = new Profile();
-		String ret = profile.read(profileID, MediaType.APPLICATION_XML);
-		logger.debug("XML :\n{}", ret);
-		ret = profile.read(profileID, MediaType.APPLICATION_JSON);
-		logger.debug("JSON : \n{}", ret);
-	}
-	
-	@Test
-	public void listRead() throws Exception {
-		Profile profile = new Profile();
-		String ret = profile.list();
-		ObjectMapper mapper = new ObjectMapper();
-		ArrayNode arrayNode = (ArrayNode) mapper.readTree(ret);
-		logger.debug("Found {} profiles", arrayNode.size());
-		Iterator<JsonNode> iterator = arrayNode.iterator();
-		while(iterator.hasNext()) {
-			String profileID = iterator.next().asText();
-			ret = profile.read(profileID, MediaType.APPLICATION_XML);
-			logger.debug("XML :\n{}", ret);
-			ret = profile.read(profileID, MediaType.APPLICATION_JSON);
-			logger.debug("JSON : \n{}", ret);
-		}
-	}
-	
-	@Test
-	public void testCreateOrUpdate() throws Exception {
-		String xml = "<metadataformat type=\"TestProfile\"><metadatafield><fieldName>test</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>1</maxOccurs><note>Test Field</note></metadatafield><metadatafield categoryref=\"FSKX_model_scope\"><fieldName>Population</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>*</maxOccurs><defaultValue/><note>The population of the model</note><tagging create=\"true\" separator=\" \">onValue</tagging><grouping create=\"true\">onFieldName_onValue</grouping></metadatafield></metadataformat>";
-		Profile profile = new Profile();
-		profile.create("TestProfile", xml);
-	}
-	
-	@Test
-	public void testDelete() throws Exception {
-		Profile profile = new Profile();
-		profile.delete("TestProfile");
-	}
-	
-	public static String PROFILE_EXAMPLE_FILENAME = "EmptyProfileExample.xml";
-	
-	public static String PROFILE_NAME_EXAMPLE = "EmptyProfile";
-	
-	@Test
-	public void testCreateUpdateDeleteGenericResource() throws Exception {
-		InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(PROFILE_EXAMPLE_FILENAME);
-		String xml = new BufferedReader(new InputStreamReader(inputStream)).lines()
-				.collect(Collectors.joining("\n"));
-		logger.debug("Body\n{}", xml);
-		Profile profile = new Profile();
-		profile.createOrUpdate(PROFILE_NAME_EXAMPLE, xml);
-		/*
-		Thread.sleep(TimeUnit.SECONDS.toMillis(30));
-		profile.createOrUpdate(PROFILE_NAME_EXAMPLE, "<metadataformat type=\"" + PROFILE_NAME_EXAMPLE + "\" />");
-		Thread.sleep(TimeUnit.SECONDS.toMillis(30));
-		profile.delete(PROFILE_NAME_EXAMPLE);
-		*/
-	}
-	
-}

src/test/java/org/gcube/gcat/rest/ResourceTest.java

@@ -9,7 +9,7 @@ import org.slf4j.LoggerFactory;
  */
 public class ResourceTest extends ContextTest {
 	
-	private static Logger logger = LoggerFactory.getLogger(ProfileTest.class);
+	private static Logger logger = LoggerFactory.getLogger(ResourceTest.class);
 	
 	// @Test
 	public void read() throws Exception {