gCubeSystem
/
gcat
15
0
Fork 0
Code Issues Pull Requests Releases 13 Wiki Activity

Fixing group and tag generation on Item Validator 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/gcat@177325 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2019-02-27 16:36:09 +00:00
parent f90b768b39
commit 3627189715
13 changed files with 172 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
distro/changelog.xml
View File

@ -2,7 +2,9 @@
<!DOCTYPE xml>
<ReleaseNotes>
<Changeset component="org.gcube.data-publishing.gcat.1-2-0" date="${buildDate}">
<Change>Saparated REST class for Profile management from the logic which effectively manage profile on IS</Change>
<Change>Separated REST class for Profile management from the logic which effectively manage profile on IS</Change>
<Change>Tags are now properly created/added according to profile definition #16182</Change>
<Change>Groups are now properly created/added according to profile definition #16183</Change>
</Changeset>
<Changeset component="org.gcube.data-publishing.gcat.1-1-0" date="2019-02-26">
<Change>Added Item URL via URI Resolver in extras field #13309</Change>

100
src/main/java/org/gcube/gcat/oldutils/Validator.java
View File

@ -12,6 +12,9 @@ import java.util.Map.Entry;
import java.util.Set;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response.Status;
import org.apache.commons.lang.math.NumberUtils;
import org.gcube.common.scope.api.ScopeProvider;
@ -23,7 +26,9 @@ import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.MetadataGrouping;
import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.MetadataTagging;
import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.NamespaceCategory;
import org.gcube.gcat.persistence.ckan.CKAN;
import org.gcube.gcat.persistence.ckan.CKANGroup;
import org.gcube.gcat.persistence.ckan.CKANPackage;
import org.gcube.gcat.persistence.ckan.CKANUser;
import org.gcube.gcat.persistence.ckan.CKANUtility;
import org.gcube.gcat.profile.MetadataUtility;
import org.gcube.gcat.social.SocialService;
@ -50,29 +55,31 @@ public class Validator {
private static final SimpleDateFormat DATE_SIMPLE = new SimpleDateFormat("yyyy-MM-dd");
private static final SimpleDateFormat DATE_HOUR_MINUTES = new SimpleDateFormat("yyyy-MM-dd HH:mm");
/**
* This method validate the incoming json dataset wrt a metadata profile
* @param json
* @param caller
* @param profiles
* @return
* @throws Exception
*/
public static ObjectNode validateAgainstProfile(ObjectMapper mapper, ObjectNode objectNode) throws Exception {
public static final int MAX_TAG_CHARS = 100;
protected ObjectMapper mapper;
public Validator() {
this.mapper = new ObjectMapper();
}
public Validator(ObjectMapper mapper) {
this.mapper = mapper;
}
public ObjectNode validateAgainstProfile(ObjectNode objectNode) throws Exception {
ArrayNode extrasArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.EXTRAS_KEY);
ArrayNode groupsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.GROUPS_KEY);
ArrayNode tagsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.TAGS_KEY);
if(extrasArrayOriginal == null || extrasArrayOriginal.size()==0) {
throw new BadRequestException("'extras' field is missing in context where metadata profile(s) are defined!");
}
ArrayNode groupsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.GROUPS_KEY);
if(groupsArrayOriginal == null) {
groupsArrayOriginal = mapper.createArrayNode();
}
ArrayNode tagsArrayOriginal = (ArrayNode) objectNode.get(CKANPackage.TAGS_KEY);
if(tagsArrayOriginal == null) {
tagsArrayOriginal = mapper.createArrayNode();
}
@ -147,7 +154,7 @@ public class Validator {
List<CustomField> validCFs = validateAgainstMetadataField(metadataIndex, categoryIdIndex,
customFields, tagsArrayOriginal, groupsArrayOriginal, metadataField, categories,
fieldsMandatoryLowerBoundMap, fieldsMandatoryUpperBoundMap, numberFieldsMandatorySameKeyMap,
groupsToCreateAfterValidation, mapper);
groupsToCreateAfterValidation);
validatedCustomFields.addAll(validCFs);
metadataIndex++;
@ -204,9 +211,9 @@ public class Validator {
// create groups
for(String title : groupsToCreateAfterValidation) {
try {
createGroupAsSysAdmin(title, title, "");
createGroupAsSysAdmin(title);
} catch(Exception e) {
logger.error("Failed to create group with title " + title, e);
logger.trace("Failed to create group with title " + title, e);
}
}
}
@ -226,10 +233,47 @@ public class Validator {
* @return
* @throws Exception
*/
public static CkanGroup createGroupAsSysAdmin(String title, String groupName, String description) throws Exception {
return CKAN.getCatalogue().createGroup(groupName, title, description);
public void createGroupAsSysAdmin(String groupName) throws Exception {
String sysAdminAPI = CKANUtility.getSysAdminAPI();
CKANGroup ckanGroup = new CKANGroup();
ckanGroup.setApiKey(sysAdminAPI);
ckanGroup.setName(CKANGroup.getGroupName(groupName));
try {
ckanGroup.read();
} catch (WebApplicationException e) {
if(e.getResponse().getStatus()==Status.NOT_FOUND.getStatusCode()) {
ckanGroup.create();
}else {
throw e;
}
}catch (Exception e) {
throw new InternalServerErrorException(e);
}finally {
try {
addUserToGroupAsSysAdmin(groupName);
}catch (WebApplicationException e) {
throw e;
}catch (Exception e) {
throw new InternalServerErrorException(e);
}
}
}
public void addUserToGroupAsSysAdmin(String groupName) throws Exception {
String username = CKANUtility.getCKANUsername();
addUserToGroupAsSysAdmin(groupName, username);
}
public void addUserToGroupAsSysAdmin(String groupName, String username) throws Exception {
String sysAdminAPI = CKANUtility.getSysAdminAPI();
CKANUser ckanUser = new CKANUser();
ckanUser.setApiKey(sysAdminAPI);
ckanUser.setName(username);
ckanUser.addToGroup(CKANGroup.getGroupName(groupName));
}
/**
* Validate this field and generate a new value (or returns the same if there is nothing to update)
* @param metadataIndex
@ -245,11 +289,11 @@ public class Validator {
* @return
* @throws Exception
*/
private static List<CustomField> validateAgainstMetadataField(int metadataIndex, int categoryIndex,
private List<CustomField> validateAgainstMetadataField(int metadataIndex, int categoryIndex,
List<CustomField> customFields, ArrayNode tagsArrayOriginal, ArrayNode groupsArrayOriginal,
MetadataField metadataField, List<NamespaceCategory> categories,
Map<String,Integer> fieldsMandatoryLowerBoundMap, Map<String,Integer> fieldsMandatoryUpperBoundMap,
Map<String,Integer> numberFieldsMandatorySameKeyMap, List<String> groupToCreate, ObjectMapper mapper) throws Exception {
Map<String,Integer> numberFieldsMandatorySameKeyMap, List<String> groupToCreate) throws Exception {
List<CustomField> toReturn = new ArrayList<CustomField>();
String metadataFieldName = metadataField.getCategoryFieldQName(); // get the qualified one, if any
@ -263,8 +307,8 @@ public class Validator {
fieldsFoundWithThisKey++;
cf.setIndexCategory(categoryIndex);
cf.setIndexMetadataField(metadataIndex);
checkAsGroup(cf, metadataField, groupsArrayOriginal, groupToCreate, mapper);
checkAsTag(cf, metadataField, tagsArrayOriginal, mapper);
checkAsGroup(cf, metadataField, groupsArrayOriginal, groupToCreate);
checkAsTag(cf, metadataField, tagsArrayOriginal);
toReturn.add(cf);
iterator.remove();
}
@ -306,7 +350,7 @@ public class Validator {
}
public static final int MAX_TAG_CHARS = 100;
/**
* Check if a tag must be generated
@ -314,8 +358,8 @@ public class Validator {
* @param metadataField
* @param tagsArrayOriginal
*/
private static void checkAsTag(CustomField fieldToValidate, MetadataField metadataField,
ArrayNode tagsArrayOriginal, ObjectMapper mapper) {
private void checkAsTag(CustomField fieldToValidate, MetadataField metadataField,
ArrayNode tagsArrayOriginal) {
MetadataTagging tagging = metadataField.getTagging();
if(tagging != null) {
@ -358,8 +402,8 @@ public class Validator {
* @param isApplication
* @throws Exception
*/
private static void checkAsGroup(CustomField fieldToValidate, MetadataField metadataField,
ArrayNode groupsArrayOriginal, List<String> groupToCreate, ObjectMapper mapper) throws Exception {
private void checkAsGroup(CustomField fieldToValidate, MetadataField metadataField,
ArrayNode groupsArrayOriginal, List<String> groupToCreate) throws Exception {
logger.debug("Custom field is " + fieldToValidate);
logger.debug("MetadataField field is " + metadataField);
@ -422,7 +466,7 @@ public class Validator {
* @return
* @throws Exception
*/
private static void validate(CustomField fieldToValidate, MetadataField metadataField) throws Exception {
private void validate(CustomField fieldToValidate, MetadataField metadataField) throws Exception {
DataType dataType = metadataField.getDataType();
String regex = metadataField.getValidator() != null ? metadataField.getValidator().getRegularExpression()

3
src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java
View File

@ -268,6 +268,9 @@ public abstract class CKAN {
int responseCode = httpURLConnection.getResponseCode();
if(responseCode >= Status.BAD_REQUEST.getStatusCode()) {
Status status = Status.fromStatusCode(responseCode);
InputStream inputStream = httpURLConnection.getErrorStream();
StringBuilder result = getStringBuilder(inputStream);
logger.trace(result.toString());
throw new WebApplicationException(status);
}
InputStream inputStream = httpURLConnection.getInputStream();

23
src/main/java/org/gcube/gcat/persistence/ckan/CKANGroup.java
View File

@ -1,5 +1,10 @@
package org.gcube.gcat.persistence.ckan;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.WebApplicationException;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* @author Luca Frosini (ISTI - CNR)
*/
@ -31,4 +36,22 @@ public class CKANGroup extends CKAN {
PURGE = GROUP_PURGE;
}
public static String getGroupName(String name) {
return name.trim().toLowerCase().replaceAll(" ", "_");
}
public String create() throws WebApplicationException {
try {
ObjectNode objectNode = mapper.createObjectNode();
objectNode.put(NAME_KEY, CKANGroup.getGroupName(name));
objectNode.put("title", name);
objectNode.put("display_name", name);
objectNode.put("description", "");
return super.create(mapper.writeValueAsString(objectNode));
}catch (WebApplicationException e) {
throw e;
}catch (Exception e) {
throw new InternalServerErrorException(e);
}
}
}

3
src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java
View File

@ -200,7 +200,8 @@ public class CKANPackage extends CKAN {
// Validating against profiles if any
MetadataUtility metadataUtility = MetadataUtility.getInstance();
if(!metadataUtility.getMetadataProfiles().isEmpty()) {
objectNode = Validator.validateAgainstProfile(mapper, objectNode);
Validator validator = new Validator(mapper);
objectNode = validator.validateAgainstProfile(objectNode);
}
return objectNode;

19
src/main/java/org/gcube/gcat/persistence/ckan/CKANUser.java
View File

@ -1,5 +1,8 @@
package org.gcube.gcat.persistence.ckan;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.WebApplicationException;
import org.gcube.gcat.utils.RandomString;
import com.fasterxml.jackson.databind.node.ObjectNode;
@ -21,6 +24,7 @@ public class CKANUser extends CKAN {
// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.delete.user_delete
public static final String USER_DELETE = CKAN.CKAN_API_PATH + "user_delete";
public static final String ADD_USER_TO_GROUP = CKAN.CKAN_API_PATH + "member_create";
public CKANUser() {
super();
@ -51,4 +55,19 @@ public class CKANUser extends CKAN {
this.delete();
}
public void addToGroup(String groupName) throws WebApplicationException {
try {
ObjectNode objectNode = mapper.createObjectNode();
objectNode.put(ID_KEY, CKANGroup.getGroupName(groupName));
objectNode.put("object", name);
objectNode.put("object_type", "user");
objectNode.put("capacity", "member");
sendPostRequest(ADD_USER_TO_GROUP, getAsString(objectNode));
}catch (WebApplicationException e) {
throw e;
}catch (Exception e) {
throw new InternalServerErrorException(e);
}
}
}

2
src/main/java/org/gcube/gcat/persistence/ckan/CKANUtility.java
View File

@ -45,7 +45,7 @@ public class CKANUtility {
return ckanUsername;
}
protected static String getCKANUsername() {
public static String getCKANUsername() {
return getCKANUsername(ContextUtility.getUsername());
}

14
src/test/java/org/gcube/gcat/oldutils/ValidatorTest.java Normal file
View File

@ -0,0 +1,14 @@
package org.gcube.gcat.oldutils;
import org.gcube.gcat.ContextTest;
import org.junit.Test;
public class ValidatorTest extends ContextTest {
@Test
public void createGroupAsSysAdmin() throws Exception {
String groupName = "Italian";
Validator validator = new Validator();
validator.createGroupAsSysAdmin(groupName);
}
}

30
src/test/java/org/gcube/gcat/persistence/ckan/CKANGroupTest.java Normal file
View File

@ -0,0 +1,30 @@
package org.gcube.gcat.persistence.ckan;
import org.gcube.gcat.ContextTest;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CKANGroupTest extends ContextTest {
private static Logger logger = LoggerFactory.getLogger(CKANGroupTest.class);
@Test
public void read() throws Exception {
CKANGroup ckanGroup = new CKANGroup();
ckanGroup.setApiKey(CKANUtility.getSysAdminAPI());
String name = "";
ckanGroup.setName(name);
String ret = ckanGroup.read();
logger.debug("{}", ret);
}
// @Test
public void delete() throws Exception {
CKANGroup ckanGroup = new CKANGroup();
ckanGroup.setApiKey(CKANUtility.getSysAdminAPI());
String name = "";
ckanGroup.setName(name);
ckanGroup.delete(true);
}
}

6
src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java
View File

@ -27,7 +27,7 @@ public class CKANPackageTest extends ContextTest {
private static final String ITEM_NAME_VALUE = "restful_transaction_model";
private static final String LICENSE_VALUE = "CC-BY-SA-4.0";
private static final String EXTRAS_TYPE_VALUE_VALUE = "FSKXModel";
private static final String EXTRAS_TYPE_VALUE_VALUE = "TestProfile";
@Test
@ -109,8 +109,8 @@ public class CKANPackageTest extends ContextTest {
extraArrayNode.add(typeNode);
ObjectNode modelNode = mapper.createObjectNode();
modelNode.put(CKANPackage.EXTRAS_KEY_KEY, "Model Language");
modelNode.put(CKANPackage.EXTRAS_VALUE_KEY, "Other");
modelNode.put(CKANPackage.EXTRAS_KEY_KEY, "test");
modelNode.put(CKANPackage.EXTRAS_VALUE_KEY, "test");
extraArrayNode.add(modelNode);
ObjectNode populationNode = mapper.createObjectNode();

2
src/test/java/org/gcube/gcat/profile/ProfileTest.java
View File

@ -57,7 +57,7 @@ public class ProfileTest extends ContextTest {
@Test
public void testCreateOrUpdate() throws Exception {
String xml = "<metadataformat type=\"TestProfile\"><metadatafield><fieldName>test</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>1</maxOccurs><note>Test Field</note></metadatafield><metadatafield categoryref=\"FSKX_model_scope\"><fieldName>Population</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>*</maxOccurs><defaultValue/><note>The population of the model</note><tagging create=\"true\" separator=\" \">onValue</tagging><grouping create=\"true\">onFieldName_onValue</grouping></metadatafield></metadataformat>";
String xml = "<metadataformat type=\"TestProfile\"><metadatafield><fieldName>test</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>1</maxOccurs><note>Test Field</note></metadatafield><metadatafield><fieldName>Population</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>*</maxOccurs><defaultValue/><note>The population of the model</note><tagging create=\"true\" separator=\"_\">onValue</tagging><grouping create=\"true\">onFieldName_onValue</grouping></metadatafield></metadataformat>";
ISProfile profile = new ISProfile();
profile.createOrUpdate("TestProfile", xml);
}

92
src/test/java/org/gcube/gcat/rest/ProfileTest.java
View File

@ -1,92 +0,0 @@
package org.gcube.gcat.rest;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Iterator;
import java.util.stream.Collectors;
import javax.ws.rs.core.MediaType;
import org.gcube.gcat.ContextTest;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
public class ProfileTest extends ContextTest {
private static Logger logger = LoggerFactory.getLogger(ProfileTest.class);
@Test
public void list() throws Exception {
Profile profile = new Profile();
String ret = profile.list();
logger.debug("{}", ret);
}
@Test
public void read() throws Exception {
String profileID = "EmptyProfile";
Profile profile = new Profile();
String ret = profile.read(profileID, MediaType.APPLICATION_XML);
logger.debug("XML :\n{}", ret);
ret = profile.read(profileID, MediaType.APPLICATION_JSON);
logger.debug("JSON : \n{}", ret);
}
@Test
public void listRead() throws Exception {
Profile profile = new Profile();
String ret = profile.list();
ObjectMapper mapper = new ObjectMapper();
ArrayNode arrayNode = (ArrayNode) mapper.readTree(ret);
logger.debug("Found {} profiles", arrayNode.size());
Iterator<JsonNode> iterator = arrayNode.iterator();
while(iterator.hasNext()) {
String profileID = iterator.next().asText();
ret = profile.read(profileID, MediaType.APPLICATION_XML);
logger.debug("XML :\n{}", ret);
ret = profile.read(profileID, MediaType.APPLICATION_JSON);
logger.debug("JSON : \n{}", ret);
}
}
@Test
public void testCreateOrUpdate() throws Exception {
String xml = "<metadataformat type=\"TestProfile\"><metadatafield><fieldName>test</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>1</maxOccurs><note>Test Field</note></metadatafield><metadatafield categoryref=\"FSKX_model_scope\"><fieldName>Population</fieldName><mandatory>false</mandatory><dataType>String</dataType><maxOccurs>*</maxOccurs><defaultValue/><note>The population of the model</note><tagging create=\"true\" separator=\" \">onValue</tagging><grouping create=\"true\">onFieldName_onValue</grouping></metadatafield></metadataformat>";
Profile profile = new Profile();
profile.create("TestProfile", xml);
}
@Test
public void testDelete() throws Exception {
Profile profile = new Profile();
profile.delete("TestProfile");
}
public static String PROFILE_EXAMPLE_FILENAME = "EmptyProfileExample.xml";
public static String PROFILE_NAME_EXAMPLE = "EmptyProfile";
@Test
public void testCreateUpdateDeleteGenericResource() throws Exception {
InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(PROFILE_EXAMPLE_FILENAME);
String xml = new BufferedReader(new InputStreamReader(inputStream)).lines()
.collect(Collectors.joining("\n"));
logger.debug("Body\n{}", xml);
Profile profile = new Profile();
profile.createOrUpdate(PROFILE_NAME_EXAMPLE, xml);
/*
Thread.sleep(TimeUnit.SECONDS.toMillis(30));
profile.createOrUpdate(PROFILE_NAME_EXAMPLE, "<metadataformat type=\"" + PROFILE_NAME_EXAMPLE + "\" />");
Thread.sleep(TimeUnit.SECONDS.toMillis(30));
profile.delete(PROFILE_NAME_EXAMPLE);
*/
}
}

2
src/test/java/org/gcube/gcat/rest/ResourceTest.java
View File

@ -9,7 +9,7 @@ import org.slf4j.LoggerFactory;
*/
public class ResourceTest extends ContextTest {
private static Logger logger = LoggerFactory.getLogger(ProfileTest.class);
private static Logger logger = LoggerFactory.getLogger(ResourceTest.class);
// @Test
public void read() throws Exception {