From 3627189715222dbc1c2b82ae8dd49cec2a82dfb2 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Wed, 27 Feb 2019 16:36:09 +0000 Subject: [PATCH] Fixing group and tag generation on Item Validator git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/gcat@177325 82a268e6-3cf1-43bd-a215-b396298e98cf --- distro/changelog.xml | 4 +- .../org/gcube/gcat/oldutils/Validator.java | 100 +++++++++++++----- .../org/gcube/gcat/persistence/ckan/CKAN.java | 3 + .../gcat/persistence/ckan/CKANGroup.java | 23 ++++ .../gcat/persistence/ckan/CKANPackage.java | 3 +- .../gcube/gcat/persistence/ckan/CKANUser.java | 19 ++++ .../gcat/persistence/ckan/CKANUtility.java | 2 +- .../gcube/gcat/oldutils/ValidatorTest.java | 14 +++ .../gcat/persistence/ckan/CKANGroupTest.java | 30 ++++++ .../persistence/ckan/CKANPackageTest.java | 6 +- .../org/gcube/gcat/profile/ProfileTest.java | 2 +- .../java/org/gcube/gcat/rest/ProfileTest.java | 92 ---------------- .../org/gcube/gcat/rest/ResourceTest.java | 2 +- 13 files changed, 172 insertions(+), 128 deletions(-) create mode 100644 src/test/java/org/gcube/gcat/oldutils/ValidatorTest.java create mode 100644 src/test/java/org/gcube/gcat/persistence/ckan/CKANGroupTest.java delete mode 100644 src/test/java/org/gcube/gcat/rest/ProfileTest.java diff --git a/distro/changelog.xml b/distro/changelog.xml index b73eb91..5ecf3e2 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -2,7 +2,9 @@ - Saparated REST class for Profile management from the logic which effectively manage profile on IS + Separated REST class for Profile management from the logic which effectively manage profile on IS + Tags are now properly created/added according to profile definition #16182 + Groups are now properly created/added according to profile definition #16183 Added Item URL via URI Resolver in extras field #13309 diff --git a/src/main/java/org/gcube/gcat/oldutils/Validator.java b/src/main/java/org/gcube/gcat/oldutils/Validator.java index d405a16..f53275f 100644 --- a/src/main/java/org/gcube/gcat/oldutils/Validator.java +++ b/src/main/java/org/gcube/gcat/oldutils/Validator.java @@ -12,6 +12,9 @@ import java.util.Map.Entry; import java.util.Set; import javax.ws.rs.BadRequestException; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response.Status; import org.apache.commons.lang.math.NumberUtils; import org.gcube.common.scope.api.ScopeProvider; @@ -23,7 +26,9 @@ import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.MetadataGrouping; import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.MetadataTagging; import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.NamespaceCategory; import org.gcube.gcat.persistence.ckan.CKAN; +import org.gcube.gcat.persistence.ckan.CKANGroup; import org.gcube.gcat.persistence.ckan.CKANPackage; +import org.gcube.gcat.persistence.ckan.CKANUser; import org.gcube.gcat.persistence.ckan.CKANUtility; import org.gcube.gcat.profile.MetadataUtility; import org.gcube.gcat.social.SocialService; @@ -50,29 +55,31 @@ public class Validator { private static final SimpleDateFormat DATE_SIMPLE = new SimpleDateFormat("yyyy-MM-dd"); private static final SimpleDateFormat DATE_HOUR_MINUTES = new SimpleDateFormat("yyyy-MM-dd HH:mm"); - /** - * This method validate the incoming json dataset wrt a metadata profile - * @param json - * @param caller - * @param profiles - * @return - * @throws Exception - */ - public static ObjectNode validateAgainstProfile(ObjectMapper mapper, ObjectNode objectNode) throws Exception { - + public static final int MAX_TAG_CHARS = 100; + + protected ObjectMapper mapper; + + public Validator() { + this.mapper = new ObjectMapper(); + } + + public Validator(ObjectMapper mapper) { + this.mapper = mapper; + } + + public ObjectNode validateAgainstProfile(ObjectNode objectNode) throws Exception { ArrayNode extrasArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.EXTRAS_KEY); - ArrayNode groupsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.GROUPS_KEY); - ArrayNode tagsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.TAGS_KEY); - if(extrasArrayOriginal == null || extrasArrayOriginal.size()==0) { throw new BadRequestException("'extras' field is missing in context where metadata profile(s) are defined!"); } + ArrayNode groupsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.GROUPS_KEY); if(groupsArrayOriginal == null) { groupsArrayOriginal = mapper.createArrayNode(); } + ArrayNode tagsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.TAGS_KEY); if(tagsArrayOriginal == null) { tagsArrayOriginal = mapper.createArrayNode(); } @@ -147,7 +154,7 @@ public class Validator { List validCFs = validateAgainstMetadataField(metadataIndex, categoryIdIndex, customFields, tagsArrayOriginal, groupsArrayOriginal, metadataField, categories, fieldsMandatoryLowerBoundMap, fieldsMandatoryUpperBoundMap, numberFieldsMandatorySameKeyMap, - groupsToCreateAfterValidation, mapper); + groupsToCreateAfterValidation); validatedCustomFields.addAll(validCFs); metadataIndex++; @@ -204,9 +211,9 @@ public class Validator { // create groups for(String title : groupsToCreateAfterValidation) { try { - createGroupAsSysAdmin(title, title, ""); + createGroupAsSysAdmin(title); } catch(Exception e) { - logger.error("Failed to create group with title " + title, e); + logger.trace("Failed to create group with title " + title, e); } } } @@ -226,10 +233,47 @@ public class Validator { * @return * @throws Exception */ - public static CkanGroup createGroupAsSysAdmin(String title, String groupName, String description) throws Exception { - return CKAN.getCatalogue().createGroup(groupName, title, description); + public void createGroupAsSysAdmin(String groupName) throws Exception { + String sysAdminAPI = CKANUtility.getSysAdminAPI(); + CKANGroup ckanGroup = new CKANGroup(); + ckanGroup.setApiKey(sysAdminAPI); + ckanGroup.setName(CKANGroup.getGroupName(groupName)); + try { + ckanGroup.read(); + } catch (WebApplicationException e) { + if(e.getResponse().getStatus()==Status.NOT_FOUND.getStatusCode()) { + ckanGroup.create(); + }else { + throw e; + } + }catch (Exception e) { + throw new InternalServerErrorException(e); + }finally { + try { + addUserToGroupAsSysAdmin(groupName); + }catch (WebApplicationException e) { + throw e; + }catch (Exception e) { + throw new InternalServerErrorException(e); + } + } } + public void addUserToGroupAsSysAdmin(String groupName) throws Exception { + String username = CKANUtility.getCKANUsername(); + addUserToGroupAsSysAdmin(groupName, username); + } + + + public void addUserToGroupAsSysAdmin(String groupName, String username) throws Exception { + String sysAdminAPI = CKANUtility.getSysAdminAPI(); + CKANUser ckanUser = new CKANUser(); + ckanUser.setApiKey(sysAdminAPI); + ckanUser.setName(username); + ckanUser.addToGroup(CKANGroup.getGroupName(groupName)); + } + + /** * Validate this field and generate a new value (or returns the same if there is nothing to update) * @param metadataIndex @@ -245,11 +289,11 @@ public class Validator { * @return * @throws Exception */ - private static List validateAgainstMetadataField(int metadataIndex, int categoryIndex, + private List validateAgainstMetadataField(int metadataIndex, int categoryIndex, List customFields, ArrayNode tagsArrayOriginal, ArrayNode groupsArrayOriginal, MetadataField metadataField, List categories, Map fieldsMandatoryLowerBoundMap, Map fieldsMandatoryUpperBoundMap, - Map numberFieldsMandatorySameKeyMap, List groupToCreate, ObjectMapper mapper) throws Exception { + Map numberFieldsMandatorySameKeyMap, List groupToCreate) throws Exception { List toReturn = new ArrayList(); String metadataFieldName = metadataField.getCategoryFieldQName(); // get the qualified one, if any @@ -263,8 +307,8 @@ public class Validator { fieldsFoundWithThisKey++; cf.setIndexCategory(categoryIndex); cf.setIndexMetadataField(metadataIndex); - checkAsGroup(cf, metadataField, groupsArrayOriginal, groupToCreate, mapper); - checkAsTag(cf, metadataField, tagsArrayOriginal, mapper); + checkAsGroup(cf, metadataField, groupsArrayOriginal, groupToCreate); + checkAsTag(cf, metadataField, tagsArrayOriginal); toReturn.add(cf); iterator.remove(); } @@ -306,7 +350,7 @@ public class Validator { } - public static final int MAX_TAG_CHARS = 100; + /** * Check if a tag must be generated @@ -314,8 +358,8 @@ public class Validator { * @param metadataField * @param tagsArrayOriginal */ - private static void checkAsTag(CustomField fieldToValidate, MetadataField metadataField, - ArrayNode tagsArrayOriginal, ObjectMapper mapper) { + private void checkAsTag(CustomField fieldToValidate, MetadataField metadataField, + ArrayNode tagsArrayOriginal) { MetadataTagging tagging = metadataField.getTagging(); if(tagging != null) { @@ -358,8 +402,8 @@ public class Validator { * @param isApplication * @throws Exception */ - private static void checkAsGroup(CustomField fieldToValidate, MetadataField metadataField, - ArrayNode groupsArrayOriginal, List groupToCreate, ObjectMapper mapper) throws Exception { + private void checkAsGroup(CustomField fieldToValidate, MetadataField metadataField, + ArrayNode groupsArrayOriginal, List groupToCreate) throws Exception { logger.debug("Custom field is " + fieldToValidate); logger.debug("MetadataField field is " + metadataField); @@ -422,7 +466,7 @@ public class Validator { * @return * @throws Exception */ - private static void validate(CustomField fieldToValidate, MetadataField metadataField) throws Exception { + private void validate(CustomField fieldToValidate, MetadataField metadataField) throws Exception { DataType dataType = metadataField.getDataType(); String regex = metadataField.getValidator() != null ? metadataField.getValidator().getRegularExpression() diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java index 3bb8ce5..08f5209 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java @@ -268,6 +268,9 @@ public abstract class CKAN { int responseCode = httpURLConnection.getResponseCode(); if(responseCode >= Status.BAD_REQUEST.getStatusCode()) { Status status = Status.fromStatusCode(responseCode); + InputStream inputStream = httpURLConnection.getErrorStream(); + StringBuilder result = getStringBuilder(inputStream); + logger.trace(result.toString()); throw new WebApplicationException(status); } InputStream inputStream = httpURLConnection.getInputStream(); diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANGroup.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANGroup.java index 404f8c9..a24932b 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANGroup.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANGroup.java @@ -1,5 +1,10 @@ package org.gcube.gcat.persistence.ckan; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.WebApplicationException; + +import com.fasterxml.jackson.databind.node.ObjectNode; + /** * @author Luca Frosini (ISTI - CNR) */ @@ -31,4 +36,22 @@ public class CKANGroup extends CKAN { PURGE = GROUP_PURGE; } + public static String getGroupName(String name) { + return name.trim().toLowerCase().replaceAll(" ", "_"); + } + + public String create() throws WebApplicationException { + try { + ObjectNode objectNode = mapper.createObjectNode(); + objectNode.put(NAME_KEY, CKANGroup.getGroupName(name)); + objectNode.put("title", name); + objectNode.put("display_name", name); + objectNode.put("description", ""); + return super.create(mapper.writeValueAsString(objectNode)); + }catch (WebApplicationException e) { + throw e; + }catch (Exception e) { + throw new InternalServerErrorException(e); + } + } } diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java index 96b6d83..c05c08f 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java @@ -200,7 +200,8 @@ public class CKANPackage extends CKAN { // Validating against profiles if any MetadataUtility metadataUtility = MetadataUtility.getInstance(); if(!metadataUtility.getMetadataProfiles().isEmpty()) { - objectNode = Validator.validateAgainstProfile(mapper, objectNode); + Validator validator = new Validator(mapper); + objectNode = validator.validateAgainstProfile(objectNode); } return objectNode; diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANUser.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANUser.java index 257c214..7740273 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANUser.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANUser.java @@ -1,5 +1,8 @@ package org.gcube.gcat.persistence.ckan; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.WebApplicationException; + import org.gcube.gcat.utils.RandomString; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -21,6 +24,7 @@ public class CKANUser extends CKAN { // see http://docs.ckan.org/en/latest/api/#ckan.logic.action.delete.user_delete public static final String USER_DELETE = CKAN.CKAN_API_PATH + "user_delete"; + public static final String ADD_USER_TO_GROUP = CKAN.CKAN_API_PATH + "member_create"; public CKANUser() { super(); @@ -51,4 +55,19 @@ public class CKANUser extends CKAN { this.delete(); } + + public void addToGroup(String groupName) throws WebApplicationException { + try { + ObjectNode objectNode = mapper.createObjectNode(); + objectNode.put(ID_KEY, CKANGroup.getGroupName(groupName)); + objectNode.put("object", name); + objectNode.put("object_type", "user"); + objectNode.put("capacity", "member"); + sendPostRequest(ADD_USER_TO_GROUP, getAsString(objectNode)); + }catch (WebApplicationException e) { + throw e; + }catch (Exception e) { + throw new InternalServerErrorException(e); + } + } } \ No newline at end of file diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANUtility.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANUtility.java index 7f6b956..bfcd871 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANUtility.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANUtility.java @@ -45,7 +45,7 @@ public class CKANUtility { return ckanUsername; } - protected static String getCKANUsername() { + public static String getCKANUsername() { return getCKANUsername(ContextUtility.getUsername()); } diff --git a/src/test/java/org/gcube/gcat/oldutils/ValidatorTest.java b/src/test/java/org/gcube/gcat/oldutils/ValidatorTest.java new file mode 100644 index 0000000..8a8f3a2 --- /dev/null +++ b/src/test/java/org/gcube/gcat/oldutils/ValidatorTest.java @@ -0,0 +1,14 @@ +package org.gcube.gcat.oldutils; + +import org.gcube.gcat.ContextTest; +import org.junit.Test; + +public class ValidatorTest extends ContextTest { + + @Test + public void createGroupAsSysAdmin() throws Exception { + String groupName = "Italian"; + Validator validator = new Validator(); + validator.createGroupAsSysAdmin(groupName); + } +} diff --git a/src/test/java/org/gcube/gcat/persistence/ckan/CKANGroupTest.java b/src/test/java/org/gcube/gcat/persistence/ckan/CKANGroupTest.java new file mode 100644 index 0000000..0c1e7f6 --- /dev/null +++ b/src/test/java/org/gcube/gcat/persistence/ckan/CKANGroupTest.java @@ -0,0 +1,30 @@ +package org.gcube.gcat.persistence.ckan; + +import org.gcube.gcat.ContextTest; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CKANGroupTest extends ContextTest { + + private static Logger logger = LoggerFactory.getLogger(CKANGroupTest.class); + + @Test + public void read() throws Exception { + CKANGroup ckanGroup = new CKANGroup(); + ckanGroup.setApiKey(CKANUtility.getSysAdminAPI()); + String name = ""; + ckanGroup.setName(name); + String ret = ckanGroup.read(); + logger.debug("{}", ret); + } + + // @Test + public void delete() throws Exception { + CKANGroup ckanGroup = new CKANGroup(); + ckanGroup.setApiKey(CKANUtility.getSysAdminAPI()); + String name = ""; + ckanGroup.setName(name); + ckanGroup.delete(true); + } +} diff --git a/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java b/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java index ae53a00..a2687bf 100644 --- a/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java +++ b/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java @@ -27,7 +27,7 @@ public class CKANPackageTest extends ContextTest { private static final String ITEM_NAME_VALUE = "restful_transaction_model"; private static final String LICENSE_VALUE = "CC-BY-SA-4.0"; - private static final String EXTRAS_TYPE_VALUE_VALUE = "FSKXModel"; + private static final String EXTRAS_TYPE_VALUE_VALUE = "TestProfile"; @Test @@ -109,8 +109,8 @@ public class CKANPackageTest extends ContextTest { extraArrayNode.add(typeNode); ObjectNode modelNode = mapper.createObjectNode(); - modelNode.put(CKANPackage.EXTRAS_KEY_KEY, "Model Language"); - modelNode.put(CKANPackage.EXTRAS_VALUE_KEY, "Other"); + modelNode.put(CKANPackage.EXTRAS_KEY_KEY, "test"); + modelNode.put(CKANPackage.EXTRAS_VALUE_KEY, "test"); extraArrayNode.add(modelNode); ObjectNode populationNode = mapper.createObjectNode(); diff --git a/src/test/java/org/gcube/gcat/profile/ProfileTest.java b/src/test/java/org/gcube/gcat/profile/ProfileTest.java index 33a32c5..f6e348b 100644 --- a/src/test/java/org/gcube/gcat/profile/ProfileTest.java +++ b/src/test/java/org/gcube/gcat/profile/ProfileTest.java @@ -57,7 +57,7 @@ public class ProfileTest extends ContextTest { @Test public void testCreateOrUpdate() throws Exception { - String xml = "testfalseString1Test FieldPopulationfalseString*The population of the modelonValueonFieldName_onValue"; + String xml = "testfalseString1Test FieldPopulationfalseString*The population of the modelonValueonFieldName_onValue"; ISProfile profile = new ISProfile(); profile.createOrUpdate("TestProfile", xml); } diff --git a/src/test/java/org/gcube/gcat/rest/ProfileTest.java b/src/test/java/org/gcube/gcat/rest/ProfileTest.java deleted file mode 100644 index 2b8b0af..0000000 --- a/src/test/java/org/gcube/gcat/rest/ProfileTest.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.gcube.gcat.rest; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Iterator; -import java.util.stream.Collectors; - -import javax.ws.rs.core.MediaType; - -import org.gcube.gcat.ContextTest; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; - -public class ProfileTest extends ContextTest { - - private static Logger logger = LoggerFactory.getLogger(ProfileTest.class); - - @Test - public void list() throws Exception { - Profile profile = new Profile(); - String ret = profile.list(); - logger.debug("{}", ret); - } - - - @Test - public void read() throws Exception { - String profileID = "EmptyProfile"; - Profile profile = new Profile(); - String ret = profile.read(profileID, MediaType.APPLICATION_XML); - logger.debug("XML :\n{}", ret); - ret = profile.read(profileID, MediaType.APPLICATION_JSON); - logger.debug("JSON : \n{}", ret); - } - - @Test - public void listRead() throws Exception { - Profile profile = new Profile(); - String ret = profile.list(); - ObjectMapper mapper = new ObjectMapper(); - ArrayNode arrayNode = (ArrayNode) mapper.readTree(ret); - logger.debug("Found {} profiles", arrayNode.size()); - Iterator iterator = arrayNode.iterator(); - while(iterator.hasNext()) { - String profileID = iterator.next().asText(); - ret = profile.read(profileID, MediaType.APPLICATION_XML); - logger.debug("XML :\n{}", ret); - ret = profile.read(profileID, MediaType.APPLICATION_JSON); - logger.debug("JSON : \n{}", ret); - } - } - - @Test - public void testCreateOrUpdate() throws Exception { - String xml = "testfalseString1Test FieldPopulationfalseString*The population of the modelonValueonFieldName_onValue"; - Profile profile = new Profile(); - profile.create("TestProfile", xml); - } - - @Test - public void testDelete() throws Exception { - Profile profile = new Profile(); - profile.delete("TestProfile"); - } - - public static String PROFILE_EXAMPLE_FILENAME = "EmptyProfileExample.xml"; - - public static String PROFILE_NAME_EXAMPLE = "EmptyProfile"; - - @Test - public void testCreateUpdateDeleteGenericResource() throws Exception { - InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(PROFILE_EXAMPLE_FILENAME); - String xml = new BufferedReader(new InputStreamReader(inputStream)).lines() - .collect(Collectors.joining("\n")); - logger.debug("Body\n{}", xml); - Profile profile = new Profile(); - profile.createOrUpdate(PROFILE_NAME_EXAMPLE, xml); - /* - Thread.sleep(TimeUnit.SECONDS.toMillis(30)); - profile.createOrUpdate(PROFILE_NAME_EXAMPLE, ""); - Thread.sleep(TimeUnit.SECONDS.toMillis(30)); - profile.delete(PROFILE_NAME_EXAMPLE); - */ - } - -} diff --git a/src/test/java/org/gcube/gcat/rest/ResourceTest.java b/src/test/java/org/gcube/gcat/rest/ResourceTest.java index a0a3883..af989fa 100644 --- a/src/test/java/org/gcube/gcat/rest/ResourceTest.java +++ b/src/test/java/org/gcube/gcat/rest/ResourceTest.java @@ -9,7 +9,7 @@ import org.slf4j.LoggerFactory; */ public class ResourceTest extends ContextTest { - private static Logger logger = LoggerFactory.getLogger(ProfileTest.class); + private static Logger logger = LoggerFactory.getLogger(ResourceTest.class); // @Test public void read() throws Exception {