gCubeSystem
/
grsf-publisher-ws
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added update method. Minor refactor for common code 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@151029 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Costantino Perciante 2017-07-12 16:05:35 +00:00
parent 983104aad2
commit 93b9e3a25e
14 changed files with 819 additions and 638 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java
View File

@ -28,6 +28,9 @@ public class Base {
private static Logger logger = LoggerFactory.getLogger(Base.class);
public static final String UUID_KB_KEY = "UUID Knowledge Base";
@JsonProperty("catalog_id") //used on patch/update product call
private String catalogId;
@JsonProperty("description")
private String description;
@ -68,27 +71,29 @@ public class Base {
}
/**
* @param id
* @param description
* @param license
* @param author
* @param version
* @param authorContact
* @param version
* @param maintainer
* @param maintainerContact
* @param extrasFields
* @param extrasResources
* @param uuid
*/
public Base(String description, String license, String author,
Long version, String authorContact, String maintainer,
public Base(String id, String description, String license, String author,
String authorContact, Long version, String maintainer,
String maintainerContact, Map<String, List<String>> extrasFields,
List<ResourceBean> extrasResources, String uuid) {
super();
this.catalogId = id;
this.description = description;
this.license = license;
this.author = author;
this.version = version;
this.authorContact = authorContact;
this.version = version;
this.maintainer = maintainer;
this.maintainerContact = maintainerContact;
this.extrasFields = extrasFields;
@ -96,6 +101,16 @@ public class Base {
this.uuid = uuid;
}
public String getCatalogId() {
return catalogId;
}
public void setCatalogId(String catalogId) {
this.catalogId = catalogId;
}
public String getDescription() {
return description;
}
@ -197,12 +212,12 @@ public class Base {
@Override
public String toString() {
return "Base [description=" + description + ", license=" + license
+ ", author=" + author + ", version=" + version
+ ", authorContact=" + authorContact + ", maintainer="
return "Base [catalogId=" + catalogId + ", description=" + description + ", license="
+ license + ", author=" + author + ", authorContact="
+ authorContact + ", version=" + version + ", maintainer="
+ maintainer + ", maintainerContact=" + maintainerContact
+ ", extrasFields=" + extrasFields + ", extrasResources="
+ extrasResources + ", uuid=" + uuid + "]";
}
}
}

8
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java
View File

@ -24,6 +24,7 @@ public class Common extends Base{
public static final String GRSF_TYPE_KEY = "GRSF type"; // stock, fishery
public static final String SOURCE_KEY = "Source"; // in case it is a RAM/FIRMS/FishSource record it is not added
// it is added in case of GRSF record
@JsonProperty("data_owner")
@CustomField(key="Data owner")
@ -50,13 +51,12 @@ public class Common extends Base{
@JsonProperty("traceability_flag")
@CustomField(key="Traceability Flag")
@Group(condition="true", groupNameOverValue="traceability-flag") // record is added to group if Traceability Flag is true
@Group(condition="true", groupNameOverValue="traceability-flag") // record is added to group traceability-flag if Traceability Flag is true
private Boolean traceabilityFlag;
@JsonProperty("status")
@CustomField(key="Status")
@Group
// @Tag
private Status status;
@JsonProperty("reporting_year")
@ -70,8 +70,6 @@ public class Common extends Base{
// automatically compiled
@JsonProperty("grsf_type")
@CustomField(key=GRSF_TYPE_KEY)
//@Tag
//@Group
private String grsfType;
// automatically compiled
@ -274,4 +272,4 @@ public class Common extends Base{
+ ", species=" + species + "]";
}
}
}

7
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/FisheryRecord.java
View File

@ -33,7 +33,6 @@ public class FisheryRecord extends Common{
@JsonProperty("fishing_area")
@CustomField(key="Fishing area")
//@Tag
private List<String> fishingArea;
@JsonProperty("exploited_stocks")
@ -42,7 +41,6 @@ public class FisheryRecord extends Common{
@JsonProperty("management_entity")
@CustomField(key="Management entity")
//@Tag
private List<String> managementEntity;
@JsonProperty("jurisdiction_area")
@ -51,7 +49,6 @@ public class FisheryRecord extends Common{
private List<String> jurisdictionArea;
@JsonProperty("production_system_type")
//@Group
@Tag
@CustomField(key="Production system type")
private List<Production_System_Type> productionSystemType;
@ -62,7 +59,6 @@ public class FisheryRecord extends Common{
private List<String> flagState;
@JsonProperty("fishing_gear")
//@Tag
@CustomField(key="Fishing gear")
private List<String> fishingGear;
@ -76,7 +72,6 @@ public class FisheryRecord extends Common{
@JsonProperty("type")
@CustomField(key="Type")
//@Tag
@Group
private Fishery_Type type;
@ -229,4 +224,4 @@ public class FisheryRecord extends Common{
+ ", type=" + type + "]";
}
}
}

3
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/output/ResponseBean.java
View File

@ -18,7 +18,6 @@ public class ResponseBean{
* @param success
* @param message
* @param result
* @param help
*/
public ResponseBean(boolean success, String message, Object result) {
super();
@ -62,4 +61,4 @@ public class ResponseBean{
return "ResponseBean [success=" + success
+ ", message=" + message + ", result=" + result + "]";
}
}
}

6
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/output/ResponseCreationBean.java
View File

@ -3,7 +3,7 @@ package org.gcube.data_catalogue.grsf_publish_ws.json.output;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A bean used to reply to a product creation method.
* A bean used to reply to a product creation/patch method.
* @author Costantino Perciante at ISTI-CNR
*/
public class ResponseCreationBean {
@ -30,7 +30,7 @@ public class ResponseCreationBean {
* @param productUrl
* @param error
*/
public ResponseCreationBean(String id, String kbUuid, String productUrl,
public ResponseCreationBean(String id, String kbUuid, String itemUrl,
String error) {
super();
this.id = id;
@ -77,4 +77,4 @@ public class ResponseCreationBean {
+ ", itemUrl=" + itemUrl + ", error=" + error + "]";
}
}
}

221
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java
View File

@ -3,11 +3,16 @@ package org.gcube.data_catalogue.grsf_publish_ws.services;
import java.beans.PropertyDescriptor;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import javax.servlet.ServletContext;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CkanResource;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField;
@ -20,23 +25,35 @@ import org.gcube.data_catalogue.grsf_publish_ws.json.input.RefersToBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Resource;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.TimeSeriesBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseCreationBean;
import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Status;
import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.AssociationToGroupThread;
import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.ManageTimeSeriesThread;
import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.WritePostCatalogueManagerThread;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean;
import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.model.CkanOrganization;
/**
* Services common utils.
* @author Costantino Perciante at ISTI-CNR
*/
@SuppressWarnings({"rawtypes", "unchecked"})
public class CommonServiceUtils {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(CommonServiceUtils.class);
public static final String DEFAULT_LICENSE = "CC-BY-SA-4.0";
public static final String GRSF_GROUP_NAME = "grsf-group";
private static final int TIME_SERIES_TAKE_LAST_VALUES = 5;
private static final String REGEX_TAGS = "[^\\s\\w-_.]";
public static final String SYSTEM_TYPE = "system:type";
// item url property
public static final String ITEM_URL_FIELD = "UUID";
@ -110,16 +127,23 @@ public class CommonServiceUtils {
/**
* Parse the record to look up tags, groups and resources
* @param tags
* @param skipTags
* @param groups
* @param skipGroups
* @param resources
* @param skipResources
* @param extras
* @param record
* @param username
* @param resources
* @param source
*/
public static void getTagsGroupsResourcesExtrasByRecord(
Set<String> tags,
boolean skipTags,
Set<String> groups,
boolean skipGroups,
List<ResourceBean> resources,
boolean skipResources,
Map<String, List<String>> extras,
Base record,
String username,
@ -133,18 +157,23 @@ public class CommonServiceUtils {
if(!skipTags)
getTagsByField(field, current, record, tags);
getGroupsByField(field, current, record, groups, source);
if(!skipGroups)
getGroupsByField(field, current, record, groups, source);
getExtrasByField(field, current, record, extras);
getResourcesByField(field, current, record, username, resources);
if(!skipResources)
getResourcesByField(field, current, record, username, resources);
}
}
while((current = current.getSuperclass())!=null); // start from the inherited class up to the Object.class
logger.info("Tags are " + tags);
logger.info("Groups are " + groups);
logger.info("Extras are " + extras);
logger.info("Resources without timeseries are " + resources);
logger.debug("Tags are " + tags);
logger.debug("Groups are " + groups);
logger.debug("Extras are " + extras);
logger.debug("Resources without timeseries are " + resources);
}
/**
@ -211,9 +240,9 @@ public class CommonServiceUtils {
// else add all the available elements
for (int i = 0; i < asList.size(); i++) {
boolean match = conditionToCheck.equals("") ? true : asList.get(i).toString().trim().matches(conditionToCheck);
boolean match = conditionToCheck.isEmpty() ? true : asList.get(i).toString().trim().matches(conditionToCheck);
if(match){
String groupName = groupNameOverValue.equals("") ?
String groupName = groupNameOverValue.isEmpty() ?
HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + asList.get(i).toString().trim()) :
source.toString().toLowerCase() + "-" + groupNameOverValue;
groups.add(groupName);
@ -224,10 +253,10 @@ public class CommonServiceUtils {
}else{
// also convert to the group name that should be on ckan
boolean match = conditionToCheck.equals("") ? true : f.toString().trim().matches(conditionToCheck);
boolean match = conditionToCheck.isEmpty() ? true : f.toString().trim().matches(conditionToCheck);
if(match){
String groupName = groupNameOverValue.equals("") ?
String groupName = groupNameOverValue.isEmpty() ?
HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + f.toString().trim()) :
source.toString().toLowerCase() + "-" + groupNameOverValue;
groups.add(groupName);
@ -236,18 +265,6 @@ public class CommonServiceUtils {
}
}
// // check if the field is an enumerator, and the enum class is also annotated with @Group
// if(field.getType().isEnum() && field.getType().isAnnotationPresent(Group.class)){
//
// logger.info("Class " + field.getClass().getSimpleName() + " has annotation @Group");
//
// // extract the name from the enum class and add it to the groups
// // also convert to the group name that should be on ckan
// String groupName = HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + field.getType().getSimpleName());
// groups.add(groupName);
//
// }
}catch(Exception e){
logger.error("Failed to read value for field " + field.getName() + " skipping", e);
}
@ -347,4 +364,158 @@ public class CommonServiceUtils {
}
}
}
/**
* Evaluate if the user has the admin role
* Throws exception if he/she doesn't
*/
public static void hasAdminRole(String username, DataCatalogue catalogue, String apiKey, String organization) throws Exception{
String role = null;
Iterator<Entry<CkanOrganization, RolesCkanGroupOrOrg>> roles = catalogue.getUserRoleByOrganization(username, apiKey).get(organization).entrySet().iterator();
while (roles.hasNext()) {
Map.Entry<CkanOrganization, RolesCkanGroupOrOrg> entry = (Map.Entry<CkanOrganization, RolesCkanGroupOrOrg>) roles
.next();
role = RolesCkanGroupOrOrg.convertToCkanCapacity(entry.getValue());
}
logger.info("Role of the user " + username + " is " + role + " in " + organization);
if(role == null || role.isEmpty() || !role.equalsIgnoreCase(RolesCkanGroupOrOrg.ADMIN.toString()))
throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-Administrator role!");
}
/**
* Check this record's name
* @param futureName
* @param catalogue
* @throws Exception on name check
*/
public static void checkName(String futureName, DataCatalogue catalogue) throws Exception {
if(!HelperMethods.isNameValid(futureName)){
throw new Exception("The 'uuid_knowledge_base' must contain only alphanumeric characters, and symbols like '.' or '_', '-'");
}else{
logger.debug("Checking if such name [" + futureName + "] doesn't exist ...");
boolean alreadyExists = catalogue.existProductWithNameOrId(futureName);
if(alreadyExists){
logger.debug("A product with 'uuid_knowledge_base' " + futureName + " already exists");
throw new Exception("A product with 'uuid_knowledge_base' " + futureName + " already exists");
}
}
}
/**
* Validate and check sources
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
* @param context
* @param contextServlet
* @param sourceInPath
* @param record
* @param resources
* @param groups
* @param customFields
* @param tags
* @param futureTitle
* @param username
* @throws Exception
*/
public static void validateRecordAndMapFields(String context, ServletContext contextServlet,
Sources sourceInPath, Common record, Product_Type productType, Set<String> tags, Map<String, List<String>> customFields, Set<String> groups, List<ResourceBean> resources, String username, String futureTitle) throws Exception {
// validate the record if it is a GRSF one and set the record type and in manage context
// Status field is needed only in the Manage context for GRSF records
if(context.equals((String)contextServlet.getInitParameter(HelperMethods.MANAGE_CONTEX_KEY))){
if(sourceInPath.equals(Sources.GRSF)){
//Evaluate the sources
List<Resource<Sources>> recordSources = record.getDatabaseSources();
String sources = "";
for (Resource<Sources> resource : recordSources) {
sources += resource.getName() + ", ";
}
sources = sources.endsWith(", ") ? sources.substring(0, sources.length() -2) : sources;
record.setSourceType(sources);
CommonServiceUtils.validateAggregatedRecord(record);
}
}
// set the grsf type: fishery or stock
record.setGrsfType(productType.getOrigName());
// evaluate the custom fields/tags, resources and groups
groups.add(sourceInPath.getOrigName().toLowerCase() + "-" + productType.getOrigName().toLowerCase()); //e.g. grsf-fishery
boolean skipTags = !sourceInPath.equals(Sources.GRSF); // no tags for the Original records
CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, false, resources, false, customFields, record, username, sourceInPath);
// manage the refers to
if(sourceInPath.equals(Sources.GRSF)){
List<RefersToBean> refersTo = record.getRefersTo();
if(refersTo == null || refersTo.isEmpty())
throw new Exception("refers_to is empty");
for (RefersToBean refersToBean : refersTo) {
resources.add(new ResourceBean(refersToBean.getUrl(), "Source of item " + futureTitle + " in the catalogue has id: "
+ refersToBean.getId(), "Information of a source of the item " + futureTitle, null, username, null, null));
}
}
// add the SYSTEM_TYPE
customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName()));
}
/**
* Actions to execute once the dataset has been updated or created.
* @param responseBean
* @param catalogue
* @param namespaces
* @param groups
* @param context
* @param token
* @param futureTitle
* @param authorFullname
* @param contextServlet
* @throws InterruptedException
*/
public static void actionsPostCreateOrUpdate(
String datasetId, String futureName, Common record, String apiKey, String username, String organization, String itemUrl,
ResponseCreationBean responseBean, DataCatalogue catalogue,
Map<String, String> namespaces, Set<String> groups, String context,
String token, String futureTitle, String authorFullname, ServletContext contextServlet, boolean isUpdated) throws InterruptedException {
// set info in the response bean
responseBean.setId(datasetId);
responseBean.setItemUrl(itemUrl);
responseBean.setKbUuid(record.getUuid());
// manage groups (wait thread to die: ckan doesn't support too much concurrency on same record ...)
if(!groups.isEmpty()){
logger.info("Launching thread for association to the list of groups " + groups);
AssociationToGroupThread threadGroups = new AssociationToGroupThread(new ArrayList<String>(groups), datasetId, organization, username, catalogue, apiKey);
threadGroups.start();
logger.debug("Waiting association thread to die..");
threadGroups.join();
logger.debug("Ok, it died");
}
// manage time series as resources
logger.info("Launching thread for time series handling");
new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start();
// write a post if the product has been published in grsf context
if(!isUpdated && context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY))){
new WritePostCatalogueManagerThread(
context,
token,
futureTitle,
itemUrl,
false,
new ArrayList<String>(),
authorFullname).start();
logger.info("Thread to write a post about the new product has been launched");
}
}
}

507
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java
View File

@ -29,37 +29,27 @@ import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.utils.Caller;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Base;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.DeleteProductBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.RefersToBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Resource;
import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseCreationBean;
import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources;
import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.AssociationToGroupThread;
import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.ManageTimeSeriesThread;
import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.WritePostCatalogueManagerThread;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean;
import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.model.CkanDataset;
/**
* Fishery web service methods
* @author Costantino Perciante at ISTI-CNR
* Fishery web service methods.
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
*/
@Path("{source:firms|FIRMS|ram|RAM|grsf|GRSF|FishSource|fishsource}/fishery/")
public class GrsfPublisherFisheryService {
// the default license for these records
private static final String DEFAULT_FISHERY_LICENSE = "CC-BY-SA-4.0";
// the context
@Context ServletContext contextServlet;
@ -77,13 +67,11 @@ public class GrsfPublisherFisheryService {
@Path("get-licenses")
@Produces(MediaType.APPLICATION_JSON)
public Response getLicenses(){
Status status = Status.OK;
Map<String, String> licenses = CommonServiceUtils.getLicenses();
if(licenses == null)
status = Status.INTERNAL_SERVER_ERROR;
return Response.status(status).entity(licenses).build();
}
@POST
@ -96,18 +84,16 @@ public class GrsfPublisherFisheryService {
@PathParam("source") String source)
throws ValidationException{
// retrieve context and username
Caller caller = AuthorizationProvider.instance.get();
String username = caller.getClient().getId();
String context = ScopeProvider.instance.get();
String token = SecurityTokenProvider.instance.get();
logger.info("Incoming request for creating a fishery record = " + record);
logger.info("Request comes from user " + username + " in context " + context);
logger.info("Incoming request for creating a fishery record = " + record + ".\nRequest comes from user " + username + " in context " + context);
ResponseCreationBean responseBean = new ResponseCreationBean();
Status status = Status.INTERNAL_SERVER_ERROR;
String id = "";
String id = ""; // id of the created record, if everything went ok
try{
@ -116,24 +102,19 @@ public class GrsfPublisherFisheryService {
DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context);
if(catalogue == null){
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("There was a problem while serving your request. No catalogue instance was found!");
}else{
String apiKey = catalogue.getApiKeyFromUsername(username);
String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin";
String role = catalogue.getRoleOfUserInOrganization(username, organization, apiKey);
CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization);
logger.info("Role of the user " + username + " is " + role);
// retrieve the user's email and fullname
String authorMail = HelperMethods.getUserEmail(context, token);
String authorFullname = HelperMethods.getUserFullname(context, token);
if(role == null || role.isEmpty())
throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-Administrator role!");
if(!role.equalsIgnoreCase(RolesCkanGroupOrOrg.ADMIN.toString())){
status = Status.FORBIDDEN;
throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-admin role!");
if(authorMail == null || authorFullname == null){
throw new Exception("Sorry but there was not possible to retrieve your fullname/email!");
}
// The name of the product will be the uuid of the kb. The title will be the fishery's fishery_name. Fishery has also the constraint that
@ -141,180 +122,80 @@ public class GrsfPublisherFisheryService {
String futureName = record.getUuid();
String futureTitle = record.getFisheryName();
if(!HelperMethods.isNameValid(futureName)){
// check name
CommonServiceUtils.checkName(futureName, catalogue);
status = Status.BAD_REQUEST;
throw new Exception("The 'uuid_knowledge_base' must contain only alphanumeric characters, and symbols like '.' or '_', '-'");
Map<String, List<String>> customFields = record.getExtrasFields();
Set<String> tags = new HashSet<String>();
Set<String> groups = new HashSet<String>();
List<ResourceBean> resources = record.getExtrasResources();
// validate end set sources
CommonServiceUtils.validateRecordAndMapFields(context, contextServlet, sourceInPath, record, Product_Type.FISHERY, tags, customFields, groups, resources, username, futureTitle);
// check the license id
String license = null;
if(record.getLicense() == null || record.getLicense().isEmpty())
license = CommonServiceUtils.DEFAULT_LICENSE;
else
if(HelperMethods.existsLicenseId(record.getLicense(), catalogue))
license = record.getLicense();
else throw new Exception("Please check the license id!");
long version = record.getVersion() == null ? 1 : record.getVersion();
// set the visibility of the datatest according the context
boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY));
// add the "Product URL" to the record
String itemUrl = catalogue.getUnencryptedUrlFromDatasetIdOrName(futureName);
customFields.put(CommonServiceUtils.ITEM_URL_FIELD, Arrays.asList(itemUrl));
// convert extras' keys to keys with namespace
Map<String, String> namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_FISHERY);
if(namespaces == null)
throw new Exception("Failed to retrieve the namespaces for the key fields!");
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
logger.info("Invoking create method..");
// create the product
id = catalogue.createCKanDatasetMultipleCustomFields(
apiKey,
futureTitle,
futureName,
organization,
authorFullname,
authorMail,
record.getMaintainer() == null? authorFullname : record.getMaintainer(),
record.getMaintainerContact() == null? authorMail : record.getMaintainerContact(),
version,
HelperMethods.removeHTML(record.getDescription()),
license,
new ArrayList<String>(tags),
customFields,
resources,
publicDataset);
// post actions
if(id != null){
logger.info("Created record with identifier " + id);
CommonServiceUtils.actionsPostCreateOrUpdate(
id, futureName, record, apiKey, username, organization,
itemUrl, responseBean, catalogue, namespaces, groups, context, token, futureTitle, authorFullname,
contextServlet, false);
status = Status.CREATED;
}else{
logger.debug("Checking if such name [" + futureName + "] doesn't exist yet...");
boolean alreadyExists = catalogue.existProductWithNameOrId(futureName);
if(alreadyExists){
logger.debug("A product with 'uuid_knowledge_base' " + futureName + " already exists");
status = Status.CONFLICT;
throw new Exception("A product with 'uuid_knowledge_base' " + futureName + " already exists");
}else{
// validate the record if it is a GRSF one and set the record type and in manage context
// Status field is needed only in the Manage context for GRSF records
if(context.equals((String)contextServlet.getInitParameter(HelperMethods.MANAGE_CONTEX_KEY))){
if(sourceInPath.equals(Sources.GRSF)){
//Evaluate the sources
List<Resource<Sources>> recordSources = record.getDatabaseSources();
String sources = "";
for (Resource<Sources> resource : recordSources) {
sources += resource.getName() + ", ";
}
sources = sources.endsWith(", ") ? sources.substring(0, sources.length() -2) : sources;
record.setSourceType(sources);
CommonServiceUtils.validateAggregatedRecord(record);
}
}
// set the grsf type
record.setGrsfType(Product_Type.FISHERY.getOrigName());
// product system type is a list of values for sources records, so remove it (so that no group is generated)
// if(!sourceInPath.equals(Sources.GRSF))
// record.setProductionSystemType(null);
// evaluate the custom fields/tags, resources and groups
Map<String, List<String>> customFields = record.getExtrasFields();
Set<String> tags = new HashSet<String>();
Set<String> groups = new HashSet<String>();
groups.add(sourceInPath.getOrigName().toLowerCase() + "-" + Product_Type.FISHERY.getOrigName().toLowerCase()); //e.g. grsf-fishery
List<ResourceBean> resources = record.getExtrasResources();
boolean skipTags = !sourceInPath.equals(Sources.GRSF); // no tags for the Original records
CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, resources, customFields, record, username, sourceInPath);
// manage the refers to
if(sourceInPath.equals(Sources.GRSF)){
List<RefersToBean> refersTo = record.getRefersTo();
if(refersTo == null || refersTo.isEmpty())
throw new Exception("refers_to is empty");
for (RefersToBean refersToBean : refersTo) {
resources.add(new ResourceBean(refersToBean.getUrl(), "Source of item " + futureTitle + " in the catalogue has id: "
+ refersToBean.getId(), "Information of a source of the item " + futureTitle, null, username, null, null));
}
}
// retrieve the user's email and fullname
String authorMail = HelperMethods.getUserEmail(context, token);
String authorFullname = HelperMethods.getUserFullname(context, token);
if(authorMail == null || authorFullname == null){
logger.debug("Author fullname or mail missing, cannot continue");
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("Sorry but there was not possible to retrieve your fullname/email!");
}else{
// check the license id
String license = null;
if(record.getLicense() == null || record.getLicense().isEmpty())
license = DEFAULT_FISHERY_LICENSE;
else
if(HelperMethods.existsLicenseId(record.getLicense(), catalogue))
license = record.getLicense();
else throw new Exception("Please check the license id!");
long version = record.getVersion() == null ? 1 : record.getVersion();
// set the visibility of the datatest according the context
boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY));
// add the SYSTEM_TYPE
customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName()));
// convert extras' keys to keys with namespace
Map<String, String> namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_FISHERY);
if(namespaces == null)
throw new Exception("Failed to retrieve the namespaces for the key fields!");
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
logger.info("Invoking creation method..");
// create the product
id = catalogue.createCKanDatasetMultipleCustomFields(
apiKey,
futureTitle,
futureName,
organization,
authorFullname,
authorMail,
record.getMaintainer(),
record.getMaintainerContact(),
version,
HelperMethods.removeHTML(record.getDescription()),
license,
new ArrayList<String>(tags),
customFields,
resources,
publicDataset);
if(id != null){
logger.info("Item created! Id is " + id);
responseBean.setId(id);
status = Status.CREATED;
String itemUrl = catalogue.getUnencryptedUrlFromDatasetIdOrName(futureName);
responseBean.setItemUrl(itemUrl);
responseBean.setKbUuid(record.getUuid());
// add the "Product URL" to the field
Map<String, List<String>> addField = new HashMap<String, List<String>>();
String modifiedUUIDKey = namespaces.containsKey(CommonServiceUtils.ITEM_URL_FIELD) ? namespaces.get(CommonServiceUtils.ITEM_URL_FIELD) : CommonServiceUtils.ITEM_URL_FIELD;
addField.put(modifiedUUIDKey, Arrays.asList(itemUrl));
catalogue.patchProductCustomFields(id, apiKey, addField);
if(!groups.isEmpty()){
logger.info("Launching thread for association to the list of groups " + groups);
AssociationToGroupThread threadGroups = new AssociationToGroupThread(new ArrayList<String>(groups), id, organization, username, catalogue);
threadGroups.start();
logger.info("Waiting association thread to die..");
threadGroups.join();
logger.debug("Groups-Thread died");
}
// manage time series
logger.info("Launching thread for time series handling");
new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start();
// write a post if the product has been published in grsf context
if(context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY))){
// TODO uncomment later
/*new WritePostCatalogueManagerThread(
context,
token,
futureTitle,
itemUrl,
false,
new ArrayList<String>(),
authorFullname).start();*/
logger.info("Thread to write a post about the new product has been launched");
}
}else{
throw new Exception("There was an error during the product generation, sorry");
}
}
}
throw new Exception("There was an error during the product generation, sorry");
}
}
}catch(Exception e){
logger.error("Failed to create fishery record" + e);
status = Status.INTERNAL_SERVER_ERROR;
responseBean.setError(e.getMessage());
}
@ -344,61 +225,47 @@ public class GrsfPublisherFisheryService {
DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context);
if(catalogue == null){
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("There was a problem while serving your request");
}
// Cast the source to the accepted ones
Sources sourceInPath = Sources.onDeserialize(source);
logger.info("The request is to delete a fishery object of source " + sourceInPath);
logger.debug("The request is to delete a fishery object of source " + sourceInPath);
// retrieve the catalogue instance
CkanDataset fisheryInCkan = catalogue.getDataset(recordToDelete.getId(), catalogue.getApiKeyFromUsername(username));
String apiKey = catalogue.getApiKeyFromUsername(username);
CkanDataset fisheryInCkan = catalogue.getDataset(recordToDelete.getId(), apiKey);
if(fisheryInCkan == null){
status = Status.NOT_FOUND;
throw new Exception("There was a problem while serving your request. This item was not found");
}
// check it is in the right source and it is a fishery
String grsfType = fisheryInCkan.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY);
String groupToCheck = sourceInPath.equals(Sources.GRSF) ? "grsf-group" : sourceInPath.getOrigName().toLowerCase();
String groupToCheck = sourceInPath.equals(Sources.GRSF) ? CommonServiceUtils.GRSF_GROUP_NAME : sourceInPath.getOrigName().toLowerCase();
if(catalogue.isDatasetInGroup(groupToCheck, recordToDelete.getId()) && Product_Type.FISHERY.getOrigName().equals(grsfType)){
logger.warn("Ok, this is a fishery of the right source, removing it");
boolean deleted = catalogue.deleteProduct(fisheryInCkan.getId(), catalogue.getApiKeyFromUsername(username), true);
logger.debug("Ok, this is a fishery of the right source, removing it");
boolean deleted = catalogue.deleteProduct(fisheryInCkan.getId(), apiKey, true);
if(deleted){
logger.info("Fishery DELETED AND PURGED!");
status = Status.OK;
responseBean.setId(fisheryInCkan.getId());
}
else{
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("Request failed, sorry. Unable to delete/purge the fishery");
}
}else{
status = Status.BAD_REQUEST;
throw new Exception("The id you are using doesn't belong to a Fishery item having source " + source + "!");
}
}catch(Exception e){
logger.error("Failed to delete this", e);
status = Status.INTERNAL_SERVER_ERROR;
responseBean.setError(e.getMessage());
}
return Response.status(status).entity(responseBean).build();
@ -410,16 +277,13 @@ public class GrsfPublisherFisheryService {
public Response getFisheriesIds(
@PathParam("source") String source){
// retrieve context and username
logger.info("Received call to get fisheries with source " + source);
Caller caller = AuthorizationProvider.instance.get();
String context = ScopeProvider.instance.get();
String username = caller.getClient().getId();
ResponseBean responseBean = new ResponseBean();
Status status = Status.INTERNAL_SERVER_ERROR;
logger.info("Received call to get fisheries with source " + source);
List<String> datasetsIds = new ArrayList<String>();
try{
@ -429,10 +293,7 @@ public class GrsfPublisherFisheryService {
DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context);
if(catalogue == null){
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("There was a problem while serving your request");
}
// if it is a request for GRSF records, we have Fishery - Stock groups, so it is easy.
@ -440,31 +301,24 @@ public class GrsfPublisherFisheryService {
if(sourceInPath.equals(Sources.GRSF))
datasetsIds = HelperMethods.getProductsInGroup(source + "-" + "fishery", catalogue);
else{
List<String> fullGroupListIds = HelperMethods.getProductsInGroup(source, catalogue);
String apiKey = catalogue.getApiKeyFromUsername(username);
for (String id : fullGroupListIds) {
CkanDataset dataset = catalogue.getDataset(id, catalogue.getApiKeyFromUsername(username));
CkanDataset dataset = catalogue.getDataset(id, apiKey);
if(dataset != null){
String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY);
if(grsfType.equals(Product_Type.FISHERY.getOrigName()))
datasetsIds.add(id);
}
}
}
status = Status.OK;
responseBean.setResult(datasetsIds);
responseBean.setSuccess(true);
}catch(Exception e){
logger.error("Failed to fetch this list of ids ", e);
status = Status.INTERNAL_SERVER_ERROR;
responseBean.setMessage(e.getMessage());
}
return Response.status(status).entity(responseBean).build();
@ -476,60 +330,189 @@ public class GrsfPublisherFisheryService {
public Response getCatalogueIdAndUrlFromKBID(
@QueryParam("name") String name){
// retrieve context and username
String context = ScopeProvider.instance.get();
Caller caller = AuthorizationProvider.instance.get();
String username = caller.getClient().getId();
ResponseBean responseBean = new ResponseBean();
Status status = Status.OK;
Status status = Status.INTERNAL_SERVER_ERROR;
logger.info("Received call to get the catalogue identifier for the product with name " + name);
try{
DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context);
if(catalogue == null){
throw new Exception("There was a problem while serving your request");
}
CkanDataset dataset = catalogue.getDataset(name, catalogue.getApiKeyFromUsername(username));
if(dataset != null){
Map<String, String> result = new HashMap<String, String>();
result.put("id", dataset.getId());
// retrieve the product url
Map<String, String> customFields = dataset.getExtrasAsHashMap();
boolean found = false;
Set<String> KeySet = customFields.keySet();
for (String key : KeySet) {
if(key.contains(CommonServiceUtils.ITEM_URL_FIELD) && !key.contains(Base.UUID_KB_KEY)){
result.put("url", customFields.get(key));
found = true;
break;
}
}
if(!found)
result.put("url", catalogue.getUnencryptedUrlFromDatasetIdOrName(dataset.getId()));
result.put("url", catalogue.getUnencryptedUrlFromDatasetIdOrName(dataset.getId()));
responseBean.setResult(result);
responseBean.setSuccess(true);
status = Status.OK;
}else{
responseBean.setMessage("Unable to retrieve a catalogue item with name " + name);
}
}catch(Exception e){
logger.error("Failed to retrieve this product", e);
status = Status.INTERNAL_SERVER_ERROR;
responseBean.setMessage(e.getMessage());
}
return Response.status(status).entity(responseBean).build();
}
@POST
@Path("update-product")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response updateFishery(
@NotNull(message="record cannot be null")
@Valid FisheryRecord record,
@PathParam("source") String source)
throws ValidationException{
}
Caller caller = AuthorizationProvider.instance.get();
String username = caller.getClient().getId();
String context = ScopeProvider.instance.get();
String token = SecurityTokenProvider.instance.get();
logger.info("Incoming request for updating a fishery record = " + record + ". Request comes from user " + username + " in context " + context);
ResponseCreationBean responseBean = new ResponseCreationBean();
Status status = Status.INTERNAL_SERVER_ERROR;
// catalog id must be reported
String catalogId = record.getCatalogId();
try{
if(catalogId == null || catalogId.isEmpty()){
status = Status.BAD_REQUEST;
throw new Exception("Please specify the 'catalog_id' property");
}
DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context);
if(catalogue == null){
throw new Exception("There was a problem while serving your request. No catalogue instance was found in this context!");
}else{
// get already published record and modify it
CkanDataset recordPublished = catalogue.getDataset(catalogId);
if(recordPublished == null)
throw new Exception("A record with catalogue id " + catalogId + " does not exist!");
// retrieve the user's email and fullname
String authorMail = HelperMethods.getUserEmail(context, token);
String authorFullname = HelperMethods.getUserFullname(context, token);
if(authorMail == null || authorFullname == null){
logger.debug("Author fullname or mail missing, cannot continue");
throw new Exception("Sorry but there was not possible to retrieve your fullname/email!");
}
String apiKey = catalogue.getApiKeyFromUsername(username);
String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin";
// check he/she has admin role
CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization);
// name, title, product url and are going to remain unchanged (so we keep them from the publisher record);
String name = recordPublished.getName();
String title = recordPublished.getTitle();
// Cast the source to the accepted ones
Sources sourceInPath = Sources.onDeserialize(source);
// load infos
Map<String, List<String>> customFields = record.getExtrasFields();
Set<String> tags = new HashSet<String>();
Set<String> groups = new HashSet<String>();
List<ResourceBean> resources = record.getExtrasResources();
// validate end set sources
CommonServiceUtils.validateRecordAndMapFields(
context,
contextServlet,
sourceInPath,
record,
Product_Type.FISHERY,
tags,
customFields,
groups,
resources,
username,
title);
// check the license id
String license = null;
if(record.getLicense() == null || record.getLicense().isEmpty())
license = CommonServiceUtils.DEFAULT_LICENSE;
else
if(HelperMethods.existsLicenseId(record.getLicense(), catalogue))
license = record.getLicense();
else throw new Exception("Please check the license id!");
long version = record.getVersion() == null ? 1 : record.getVersion();
// set the visibility of the datatest according the context
boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY));
// add the SYSTEM_TYPE
customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName()));
// convert extras' keys to keys with namespace
Map<String, String> namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_FISHERY);
if(namespaces == null)
throw new Exception("Failed to retrieve the namespaces for the key fields!");
// retrieve the url
String modifiedUUIDKey = namespaces.containsKey(CommonServiceUtils.ITEM_URL_FIELD) ? namespaces.get(CommonServiceUtils.ITEM_URL_FIELD) : CommonServiceUtils.ITEM_URL_FIELD;
String itemUrl = recordPublished.getExtrasAsHashMap().get(modifiedUUIDKey);
customFields.put(CommonServiceUtils.ITEM_URL_FIELD, Arrays.asList(itemUrl));
// replace fields
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
logger.info("Invoking update method..");
// update the product
String id = catalogue.updateCKanDataset(
apiKey,
catalogId,
title, name,
organization,
authorFullname,
authorMail,
record.getMaintainer(),
record.getMaintainerContact(),
version,
HelperMethods.removeHTML(record.getDescription()),
license,
new ArrayList<String>(tags),
null, // remove any previous group
customFields,
resources,
publicDataset);
if(id != null){
logger.info("Item updated!");
CommonServiceUtils.actionsPostCreateOrUpdate(
id, name, record, apiKey, username, organization,
itemUrl, responseBean, catalogue, namespaces, groups, context, token, title, authorFullname,
contextServlet, false);
status = Status.OK;
}else{
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("There was an error during the item updated, sorry");
}
}
}catch(Exception e){
logger.error("Failed to create fishery record" + e);
responseBean.setError(e.getMessage());
}
return Response.status(status).entity(responseBean).build();
}
}

494
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java
View File

@ -29,36 +29,28 @@ import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.utils.Caller;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Base;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.DeleteProductBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.RefersToBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Resource;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord;
import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseCreationBean;
import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources;
import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.AssociationToGroupThread;
import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.ManageTimeSeriesThread;
import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.WritePostCatalogueManagerThread;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean;
import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.model.CkanDataset;
/**
* Stock web service methods
* @author Costantino Perciante at ISTI-CNR
* Stock web service methods.
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
*/
@Path("{source:firms|FIRMS|ram|RAM|grsf|GRSF|FishSource|fishsource}/stock/")
public class GrsfPublisherStockService {
private static final String DEFAULT_STOCK_LICENSE = "CC-BY-SA-4.0";
// the context
@Context ServletContext contextServlet;
@ -93,227 +85,133 @@ public class GrsfPublisherStockService {
@NotNull(message="record cannot be null") @Valid StockRecord record,
@PathParam("source") String source) throws ValidationException{
// retrieve context and username
Caller caller = AuthorizationProvider.instance.get();
String username = caller.getClient().getId();
String context = ScopeProvider.instance.get();
String token = SecurityTokenProvider.instance.get();
logger.info("Incoming request for creating a stock record = " + record);
logger.info("Request coming from user " + username + " in context " + context);
logger.info("Incoming request for creating a stock record = " + record + ".\nRequest coming from user " + username + " in context " + context);
ResponseCreationBean responseBean = new ResponseCreationBean();
Status status = Status.INTERNAL_SERVER_ERROR;
String id = ""; // id of the created record, if everything went ok
try{
// Cast the source to the accepted ones
Sources sourceInPath = Sources.onDeserialize(source);
logger.info("The request is to create a stock object of source " + sourceInPath);
DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context);
if(catalogue == null){
throw new Exception("There was a problem while serving your request");
throw new Exception("There was a problem while serving your request. No catalogue instance was found!");
}else{
String apiKey = catalogue.getApiKeyFromUsername(username);
String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin";// TODO
String role = catalogue.getRoleOfUserInOrganization(username, organization, apiKey);
String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin";
logger.info("Role of the user " + username + " is " + role);
if(role == null || role.isEmpty())
throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-Administrator role!");
// check it has admin role or throw exception
CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization);
if(!role.equalsIgnoreCase(RolesCkanGroupOrOrg.ADMIN.toString())){
status = Status.FORBIDDEN;
throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-Administrator role!");
// retrieve the user's email and fullname
String authorMail = HelperMethods.getUserEmail(context, token);
String authorFullname = HelperMethods.getUserFullname(context, token);
if(authorMail == null || authorFullname == null){
throw new Exception("Sorry but it was not possible to retrieve your fullname/email!");
}
// check the record has a name, at least
String futureName = record.getUuid();
String futureTitle = record.getStockName();
if(!HelperMethods.isNameValid(futureName)){
// check name and throws exception
CommonServiceUtils.checkName(futureName, catalogue);
status = Status.BAD_REQUEST;
throw new Exception("The 'uuid_knowledge_base' must contain only alphanumeric characters, and symbols like '.' or '_', '-'");
// load other information
Map<String, List<String>> customFields = record.getExtrasFields();
Set<String> tags = new HashSet<String>();
Set<String> groups = new HashSet<String>();
List<ResourceBean> resources = record.getExtrasResources();
}else{
// validate end set sources, tags, etc
CommonServiceUtils.validateRecordAndMapFields(
context,
contextServlet,
sourceInPath,
record,
Product_Type.STOCK,
tags,
customFields,
groups,
resources,
username,
futureTitle);
logger.debug("Checking if such 'uuid_knowledge_base' [" + futureName + "] doesn't exist yet...");
boolean alreadyExist = catalogue.existProductWithNameOrId(futureName);
// check the license id
String license = null;
if(record.getLicense() == null || record.getLicense().isEmpty())
license = CommonServiceUtils.DEFAULT_LICENSE;
else
if(HelperMethods.existsLicenseId(record.getLicense(), catalogue))
license = record.getLicense();
else throw new Exception("Please check the license id!");
if(alreadyExist){
// check the version
long version = record.getVersion() == null ? 1 : record.getVersion();
logger.debug("A product with 'uuid_knowledge_base' " + futureName + " already exists");
status = Status.CONFLICT;
throw new Exception("A product with 'uuid_knowledge_base' " + futureName + " already exists");
// set the visibility of the datatest according the context
boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY));
}else{
// add the "Product URL" to the record
String itemUrl = catalogue.getUnencryptedUrlFromDatasetIdOrName(futureName);
customFields.put(CommonServiceUtils.ITEM_URL_FIELD, Arrays.asList(itemUrl));
// validate the record if it is a GRSF one and set the record type and in manage context
// Status field is needed only in the Manage context for GRSF records
if(context.equals((String)contextServlet.getInitParameter(HelperMethods.MANAGE_CONTEX_KEY))){
if(sourceInPath.equals(Sources.GRSF)){
//Evaluate the sources
List<Resource<Sources>> recordSources = record.getDatabaseSources();
String sources = "";
for (Resource<Sources> resource : recordSources) {
sources += resource.getName() + ", ";
}
sources = sources.endsWith(", ") ? sources.substring(0, sources.length() -2) : sources;
record.setSourceType(sources);
CommonServiceUtils.validateAggregatedRecord(record);
}
}
// convert extras' keys to keys with namespace
Map<String, String> namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_STOCK);
// set the grsf type
record.setGrsfType(Product_Type.STOCK.getOrigName());
if(namespaces == null)
throw new Exception("Failed to retrieve the namespaces for the key fields!");
// evaluate the custom fields/tags, resources and groups
Map<String, List<String>> customFields = record.getExtrasFields();
Set<String> tags = new HashSet<String>();
Set<String> groups = new HashSet<String>();
groups.add(sourceInPath.getOrigName().toLowerCase() + "-" + Product_Type.STOCK.getOrigName().toLowerCase()); //e.g. grsf-stock
List<ResourceBean> resources = record.getExtrasResources();
boolean skipTags = !sourceInPath.equals(Sources.GRSF); // no tags for the Original records
CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, resources, customFields, record, username, sourceInPath);
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
// manage the refers to
if(sourceInPath.equals(Sources.GRSF)){
logger.info("Invoking create method..");
List<RefersToBean> refersTo = record.getRefersTo();
if(refersTo == null || refersTo.isEmpty())
throw new Exception("refers_to is empty");
for (RefersToBean refersToBean : refersTo) {
resources.add(new ResourceBean(refersToBean.getUrl(), "Source of item " + futureTitle + " in the catalogue has id: "
+ refersToBean.getId(), "Information of a source of the item " + futureTitle, null, username, null, null));
}
// create the product
id = catalogue.createCKanDatasetMultipleCustomFields(
apiKey,
futureTitle,
futureName,
organization,
authorFullname,
authorMail,
record.getMaintainer() == null? authorFullname : record.getMaintainer(),
record.getMaintainerContact() == null? authorMail : record.getMaintainerContact(),
version,
HelperMethods.removeHTML(record.getDescription()),
license,
new ArrayList<String>(tags),
customFields,
resources,
publicDataset);
}
if(id != null){
// retrieve the user's email and fullname
String authorMail = HelperMethods.getUserEmail(context, token);
String authorFullname = HelperMethods.getUserFullname(context, token);
logger.info("Product created! Id is " + id);
CommonServiceUtils.actionsPostCreateOrUpdate(
id, futureName, record, apiKey, username, organization, itemUrl,
responseBean, catalogue, namespaces, groups, context, token,
futureTitle, authorFullname, contextServlet, false);
status = Status.CREATED;
if(authorMail == null || authorFullname == null){
logger.debug("Author fullname or mail missing, cannot continue");
responseBean.setId(null);
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("Sorry but was not possible to retrieve your fullname/email!");
}else{
// check the license id
String license = null;
if(record.getLicense() == null || record.getLicense().isEmpty())
license = DEFAULT_STOCK_LICENSE;
else
if(HelperMethods.existsLicenseId(record.getLicense(), catalogue))
license = record.getLicense();
else throw new Exception("Please check the license id!");
long version = record.getVersion() == null ? 1 : record.getVersion();
// set the visibility of the datatest according the context
boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY));
// add the SYSTEM_TYPE
customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName()));
logger.info("Invoking creation method..");
// convert extras' keys to keys with namespace
Map<String, String> namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_STOCK);
if(namespaces == null)
throw new Exception("Failed to retrieve the namespaces for the key fields!");
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
// create the product
String id = catalogue.createCKanDatasetMultipleCustomFields(
apiKey,
futureTitle,
futureName,
organization,
authorFullname,
authorMail,
record.getMaintainer(),
record.getMaintainerContact(),
version,
HelperMethods.removeHTML(record.getDescription()),
license,
new ArrayList<String>(tags),
customFields,
resources,
publicDataset);
if(id != null){
logger.info("Product created! Id is " + id);
responseBean.setId(id);
status = Status.CREATED;
String itemUrl = catalogue.getUnencryptedUrlFromDatasetIdOrName(futureName);
responseBean.setItemUrl(itemUrl);
responseBean.setKbUuid(record.getUuid());
// add the "Product URL" to the field
Map<String, List<String>> addField = new HashMap<String, List<String>>();
String modifiedUUIDKey = namespaces.containsKey(CommonServiceUtils.ITEM_URL_FIELD) ? namespaces.get(CommonServiceUtils.ITEM_URL_FIELD) : CommonServiceUtils.ITEM_URL_FIELD;
addField.put(modifiedUUIDKey, Arrays.asList(itemUrl));
catalogue.patchProductCustomFields(id, apiKey, addField);
if(!groups.isEmpty()){
logger.info("Launching thread for association to the list of groups " + groups);
AssociationToGroupThread threadGroups = new AssociationToGroupThread(new ArrayList<String>(groups), id, organization, username, catalogue);
threadGroups.start();
logger.info("Waiting association thread to die..");
threadGroups.join();
logger.debug("Ok, it died");
}
// manage time series
logger.info("Launching thread for time series handling");
new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start();
// write a post if the product has been published in grsf context
if(context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY))){
// TODO uncomment later
/*new WritePostCatalogueManagerThread(
context,
token,
futureTitle,
itemUrl,
false,
new ArrayList<String>(),
authorFullname).start();
logger.info("Thread to write a post about the new product has been launched");*/
}
}else
throw new Exception("There was an error during the product generation, sorry");
}
}
}
}else
throw new Exception("There was an error during the product generation, sorry! Unable to create the dataset");
}
// }
}catch(Exception e){
logger.error("Failed to create stock record", e);
responseBean.setError(e.getMessage());
}
return Response.status(status).entity(responseBean).build();
}
}
@DELETE
@Path("delete-product")
@ -344,11 +242,11 @@ public class GrsfPublisherStockService {
// Cast the source to the accepted ones
Sources sourceInPath = Sources.onDeserialize(source);
String apiKey = catalogue.getApiKeyFromUsername(username);
logger.info("The request is to delete a stock object of source " + sourceInPath);
// retrieve the catalogue instance
CkanDataset stockInCkan = catalogue.getDataset(recordToDelete.getId(), catalogue.getApiKeyFromUsername(username));
CkanDataset stockInCkan = catalogue.getDataset(recordToDelete.getId(), apiKey);
if(stockInCkan == null){
status = Status.NOT_FOUND;
@ -357,39 +255,28 @@ public class GrsfPublisherStockService {
// check it is in the right source and it is a fishery
String grsfType = stockInCkan.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY);
String groupToCheck = sourceInPath.equals(Sources.GRSF) ? "grsf-group" : sourceInPath.getOrigName().toLowerCase();
String groupToCheck = sourceInPath.equals(Sources.GRSF) ? CommonServiceUtils.GRSF_GROUP_NAME : sourceInPath.getOrigName().toLowerCase();
if(catalogue.isDatasetInGroup(groupToCheck, recordToDelete.getId()) && Product_Type.STOCK.getOrigName().equals(grsfType)){
logger.debug("Ok, this is a stock of the right type, removing it");
boolean deleted = catalogue.deleteProduct(stockInCkan.getId(), catalogue.getApiKeyFromUsername(username), true);
boolean deleted = catalogue.deleteProduct(stockInCkan.getId(), apiKey, true);
if(deleted){
logger.info("Stock DELETED AND PURGED!");
status = Status.OK;
responseBean.setId(stockInCkan.getId());
}
else{
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("Request failed, sorry. Unable to delete/purge the stock");
}
}else{
status = Status.BAD_REQUEST;
throw new Exception("The id you are using doesn't belong to a Stock product having source " + source + "!");
}
}catch(Exception e){
logger.error("Failed to delete this ", e);
responseBean.setError(e.getMessage());
}
return Response.status(status).entity(responseBean).build();
}
@ -403,26 +290,20 @@ public class GrsfPublisherStockService {
Caller caller = AuthorizationProvider.instance.get();
String context = ScopeProvider.instance.get();
String username = caller.getClient().getId();
Status status = Status.INTERNAL_SERVER_ERROR;
ResponseBean responseBean = new ResponseBean();
Status status = Status.OK;
// check it is a stock ...
logger.info("Received call to get stocks with source " + source);
List<String> datasetsIds = new ArrayList<String>();
try{
// Cast the source to the accepted ones
Sources sourceInPath = Sources.onDeserialize(source);
DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context);
if(catalogue == null){
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("There was a problem while serving your request");
}
// if it is a request for GRSF records, we have Fishery - Stock groups, so it is easy.
@ -430,33 +311,23 @@ public class GrsfPublisherStockService {
if(sourceInPath.equals(Sources.GRSF))
datasetsIds = HelperMethods.getProductsInGroup(source + "-" + "stock", catalogue);
else{
List<String> fullGroupListIds = HelperMethods.getProductsInGroup(source, catalogue);
for (String id : fullGroupListIds) {
CkanDataset dataset = catalogue.getDataset(id, catalogue.getApiKeyFromUsername(username));
if(dataset != null){
String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY);
if(grsfType.equals(Product_Type.STOCK.getOrigName()))
datasetsIds.add(id);
}
}
}
responseBean.setResult(datasetsIds);
responseBean.setSuccess(true);
status = Status.OK;
}catch(Exception e){
logger.error("Failed to fetch this list of ids " + source, e);
responseBean.setSuccess(false);
responseBean.setMessage(e.getMessage());
}
return Response.status(status).entity(responseBean).build();
}
@ -470,54 +341,187 @@ public class GrsfPublisherStockService {
String context = ScopeProvider.instance.get();
Caller caller = AuthorizationProvider.instance.get();
String username = caller.getClient().getId();
ResponseBean responseBean = new ResponseBean();
Status status = Status.OK;
Status status = Status.INTERNAL_SERVER_ERROR;
logger.info("Received call to get the catalogue identifier for the product with name " + name);
try{
DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context);
if(catalogue == null){
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("There was a problem while serving your request");
}
CkanDataset dataset = catalogue.getDataset(name, catalogue.getApiKeyFromUsername(username));
if(dataset != null){
Map<String, String> result = new HashMap<String, String>();
result.put("id", dataset.getId());
// retrieve the product url
Map<String, String> customFields = dataset.getExtrasAsHashMap();
boolean found = false;
Set<String> KeySet = customFields.keySet();
for (String key : KeySet) {
if(key.contains(CommonServiceUtils.ITEM_URL_FIELD) && !key.contains(Base.UUID_KB_KEY)){
result.put("url", customFields.get(key));
found = true;
break;
}
}
if(!found)
result.put("url", catalogue.getUnencryptedUrlFromDatasetIdOrName(dataset.getId()));
result.put("url", catalogue.getUnencryptedUrlFromDatasetIdOrName(dataset.getId()));
responseBean.setResult(result);
responseBean.setSuccess(true);
status = Status.OK;
}else{
responseBean.setMessage("Unable to retrieve a catalogue product with name " + name);
}
}catch(Exception e){
logger.error("Failed to retrieve this product", e);
responseBean.setSuccess(false);
responseBean.setMessage(e.getMessage());
}
return Response.status(status).entity(responseBean).build();
}
}
@POST
@Path("update-product")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response updateFishery(
@NotNull(message="record cannot be null")
@Valid FisheryRecord record,
@PathParam("source") String source)
throws ValidationException{
Caller caller = AuthorizationProvider.instance.get();
String username = caller.getClient().getId();
String context = ScopeProvider.instance.get();
String token = SecurityTokenProvider.instance.get();
logger.info("Incoming request for updating a stock record = " + record + ". Request comes from user " + username + " in context " + context);
ResponseCreationBean responseBean = new ResponseCreationBean();
Status status = Status.INTERNAL_SERVER_ERROR;
// catalog id must be reported
String catalogId = record.getCatalogId();
try{
if(catalogId == null || catalogId.isEmpty()){
status = Status.BAD_REQUEST;
throw new Exception("Please specify the 'catalog_id' property");
}
DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context);
if(catalogue == null){
throw new Exception("There was a problem while serving your request. No catalogue instance was found in this context!");
}else{
// get already published record and modify it
CkanDataset recordPublished = catalogue.getDataset(catalogId);
if(recordPublished == null)
throw new Exception("A record with catalogue id " + catalogId + " does not exist!");
// retrieve the user's email and fullname
String authorMail = HelperMethods.getUserEmail(context, token);
String authorFullname = HelperMethods.getUserFullname(context, token);
if(authorMail == null || authorFullname == null){
logger.debug("Author fullname or mail missing, cannot continue");
throw new Exception("Sorry but there was not possible to retrieve your fullname/email!");
}
String apiKey = catalogue.getApiKeyFromUsername(username);
String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin";
// check he/she has admin role
CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization);
// name, title, product url and are going to remain unchanged (so we keep them from the publisher record);
String name = recordPublished.getName();
String title = recordPublished.getTitle();
// Cast the source to the accepted ones
Sources sourceInPath = Sources.onDeserialize(source);
// load infos
Map<String, List<String>> customFields = record.getExtrasFields();
Set<String> tags = new HashSet<String>();
Set<String> groups = new HashSet<String>();
List<ResourceBean> resources = record.getExtrasResources();
// validate end set sources
CommonServiceUtils.validateRecordAndMapFields(
context,
contextServlet,
sourceInPath,
record,
Product_Type.STOCK,
tags,
customFields,
groups,
resources,
username,
title);
// check the license id
String license = null;
if(record.getLicense() == null || record.getLicense().isEmpty())
license = CommonServiceUtils.DEFAULT_LICENSE;
else
if(HelperMethods.existsLicenseId(record.getLicense(), catalogue))
license = record.getLicense();
else throw new Exception("Please check the license id!");
long version = record.getVersion() == null ? 1 : record.getVersion();
// set the visibility of the datatest according the context
boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY));
// add the SYSTEM_TYPE
customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName()));
// convert extras' keys to keys with namespace
Map<String, String> namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_FISHERY);
if(namespaces == null)
throw new Exception("Failed to retrieve the namespaces for the key fields!");
// retrieve the url
String modifiedUUIDKey = namespaces.containsKey(CommonServiceUtils.ITEM_URL_FIELD) ? namespaces.get(CommonServiceUtils.ITEM_URL_FIELD) : CommonServiceUtils.ITEM_URL_FIELD;
String itemUrl = recordPublished.getExtrasAsHashMap().get(modifiedUUIDKey);
customFields.put(CommonServiceUtils.ITEM_URL_FIELD, Arrays.asList(itemUrl));
// replace fields
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
logger.info("Invoking update method..");
// update the product
String id = catalogue.updateCKanDataset(
apiKey,
catalogId,
title, name,
organization,
authorFullname,
authorMail,
record.getMaintainer(),
record.getMaintainerContact(),
version,
HelperMethods.removeHTML(record.getDescription()),
license,
new ArrayList<String>(tags),
null, // remove any previous group
customFields,
resources,
publicDataset);
if(id != null){
logger.info("Item updated!");
CommonServiceUtils.actionsPostCreateOrUpdate(
recordPublished.getId(), name, record, apiKey, username, organization, itemUrl,
responseBean, catalogue, namespaces, groups, context, token,
title, authorFullname, contextServlet, true);
status = Status.OK;
}else{
status = Status.INTERNAL_SERVER_ERROR;
throw new Exception("There was an error during the item updated, sorry");
}
}
}catch(Exception e){
logger.error("Failed to create fishery record" + e);
responseBean.setError(e.getMessage());
}
return Response.status(status).entity(responseBean).build();
}
}

22
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVHelpers.java
View File

@ -20,6 +20,10 @@ public class CSVHelpers {
private static final String CSV_SEPARATOR = ",";
private static final String UPLOAD_LOCATION_LOCAL = System.getProperty("java.io.tmpdir");
private static final String GRSF_SUB_PATH = "GRSF_TIME_SERIES";
public static final String CSV_EXTENSION = ".csv";
private static final String YEAR_FIELD = "year";
private static final String VALUE_FIELD = "value";
private static final String UNIT_FIELD = "unit";
/**
* Write a time series to a csv file, and returns the file reference.<br>
@ -36,7 +40,7 @@ public class CSVHelpers {
}else
try{
String fileName = UPLOAD_LOCATION_LOCAL + File.separator + GRSF_SUB_PATH + File.separator + "time_series_" + System.currentTimeMillis() + ".csv";
String fileName = UPLOAD_LOCATION_LOCAL + File.separator + GRSF_SUB_PATH + File.separator + "time_series_" + System.currentTimeMillis() + CSV_EXTENSION;
File file = new File(fileName);
file.getParentFile().mkdirs();
file.createNewFile();
@ -47,16 +51,16 @@ public class CSVHelpers {
StringBuffer headerLine = new StringBuffer();
headerLine.append("year");
headerLine.append(YEAR_FIELD);
headerLine.append(CSV_SEPARATOR);
headerLine.append("value");
headerLine.append(VALUE_FIELD);
// first line is csv header, check the type of object
if(timeSeries.get(0).getUnit() != null && !timeSeries.get(0).getUnit().getClass().equals(Void.class)){
isUnitPresent = true;
headerLine.append(CSV_SEPARATOR);
headerLine.append("unit");
headerLine.append(UNIT_FIELD);
}
@ -79,13 +83,13 @@ public class CSVHelpers {
bw.write(oneLine.toString());
bw.newLine();
bw.flush();
}
// file created
logger.info("CSV file created correctly on this machine!");
bw.flush();
bw.close();
// file created
logger.debug("CSV file created correctly on this machine!");
// on exit delete it...
file.deleteOnExit();
@ -96,4 +100,4 @@ public class CSVHelpers {
return null;
}
}
}
}

113
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java
View File

@ -30,7 +30,6 @@ import org.gcube.data_catalogue.grsf_publish_ws.utils.cache.CacheImpl;
import org.gcube.data_catalogue.grsf_publish_ws.utils.cache.CacheInterface;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueFactory;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueImpl;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.Query;
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
@ -63,7 +62,7 @@ public abstract class HelperMethods {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(HelperMethods.class);
private static final String APPLICATION_ID_CATALOGUE_MANAGER = "org.gcube.datacatalogue.ProductCatalogue";
private static final String NOTIFICATION_MESSAGE = "Dear members,<br>The product 'PRODUCT_TITLE' has been just published by USER_FULLNAME.<br>You can find it here: PRODUCT_URL <br>";
private static final String NOTIFICATION_MESSAGE = "Dear members,<br>The item 'PRODUCT_TITLE' has been just published by USER_FULLNAME.<br>You can find it here: PRODUCT_URL <br>";
private static final String SOCIAL_SERVICE_APPLICATION_TOKEN = "/2/tokens/generate-application-token/";
private static final String SOCIAL_SERVICE_WRITE_APPLICATION_POST = "/2/posts/write-post-app/";
private static final String MEDIATYPE_JSON = "application/json";
@ -78,8 +77,10 @@ public abstract class HelperMethods {
private static final String PATH_SEPARATOR = "/";
// caches
private static CacheInterface<String, String> userEmailCache = new CacheImpl<String, String>(1000 * 60 * 120);
private static CacheInterface<String, String> userFullnameCache = new CacheImpl<String, String>(1000 * 60 * 120);
private static CacheInterface<String, String> userEmailCache = new CacheImpl<String, String>(1000 * 60 * 60 * 24);
private static CacheInterface<String, String> userFullnameCache = new CacheImpl<String, String>(1000 * 60 * 60 * 24);
private static CacheInterface<String, Map<String, String>> namespacesCache = new CacheImpl<String, Map<String, String>>(1000 * 60 * 60 * 24);
private static CacheInterface<String, DataCatalogue> catalogueCache = new CacheImpl<String, DataCatalogue>(1000 * 60 * 60 * 24);
/**
* Convert a group name to its id on ckan
@ -109,14 +110,18 @@ public abstract class HelperMethods {
*/
public static DataCatalogue getDataCatalogueRunningInstance(String scope){
try{
DataCatalogueImpl instance = DataCatalogueFactory.getFactory().getUtilsPerScope(scope);
return instance;
}catch(Exception e){
logger.error("Failed to instanciate data catalogue lib", e);
if(catalogueCache.get(scope) != null)
return catalogueCache.get(scope);
else{
try{
DataCatalogue instance = DataCatalogueFactory.getFactory().getUtilsPerScope(scope);
catalogueCache.insert(scope, instance);
return instance;
}catch(Exception e){
logger.error("Failed to instanciate data catalogue lib", e);
return null;
}
}
return null;
}
/**
@ -496,44 +501,50 @@ public abstract class HelperMethods {
public static Map<String, String> getFieldToFieldNameSpaceMapping(String resourceName){
Map<String, String> toReturn = new HashMap<String, String>();
try {
Query q = new QueryBox("for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+ "ApplicationProfile" + "' and $profile/Profile/Name/string() " +
" eq '" + resourceName + "'" +
"return $profile");
DiscoveryClient<String> client = client();
List<String> appProfile = client.submit(q);
if (appProfile == null || appProfile.size() == 0)
throw new Exception("Your applicationProfile is not registered in the infrastructure");
else {
String elem = appProfile.get(0);
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Node node = docBuilder.parse(new InputSource(new StringReader(elem))).getDocumentElement();
XPathHelper helper = new XPathHelper(node);
NodeList nodeListKeys = helper.evaluateForNodes("//originalKey");
NodeList nodeListModifiedKeys = helper.evaluateForNodes("//modifiedKey");
int sizeKeys = nodeListKeys != null ? nodeListKeys.getLength() : 0;
int sizeKeysModifed = nodeListModifiedKeys != null ? nodeListModifiedKeys.getLength() : 0;
if(sizeKeys != sizeKeysModifed)
throw new Exception("Malformed XML");
logger.debug("Size is " + sizeKeys);
for (int i = 0; i < sizeKeys; i++) {
toReturn.put(nodeListKeys.item(i).getTextContent(), nodeListModifiedKeys.item(i).getTextContent());
}
}
} catch (Exception e) {
logger.error("Error while trying to fetch applicationProfile profile from the infrastructure", e);
return null;
// check if data are in cache
if(namespacesCache.get(resourceName) != null){
return namespacesCache.get(resourceName);
}
else{
try {
Query q = new QueryBox("for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+ "ApplicationProfile" + "' and $profile/Profile/Name/string() " +
" eq '" + resourceName + "'" +
"return $profile");
logger.debug("Map is " + toReturn);
return toReturn;
DiscoveryClient<String> client = client();
List<String> appProfile = client.submit(q);
if (appProfile == null || appProfile.size() == 0)
throw new Exception("Your applicationProfile is not registered in the infrastructure");
else {
String elem = appProfile.get(0);
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Node node = docBuilder.parse(new InputSource(new StringReader(elem))).getDocumentElement();
XPathHelper helper = new XPathHelper(node);
NodeList nodeListKeys = helper.evaluateForNodes("//originalKey");
NodeList nodeListModifiedKeys = helper.evaluateForNodes("//modifiedKey");
int sizeKeys = nodeListKeys != null ? nodeListKeys.getLength() : 0;
int sizeKeysModifed = nodeListModifiedKeys != null ? nodeListModifiedKeys.getLength() : 0;
if(sizeKeys != sizeKeysModifed)
throw new Exception("Malformed XML");
logger.debug("Size is " + sizeKeys);
for (int i = 0; i < sizeKeys; i++) {
toReturn.put(nodeListKeys.item(i).getTextContent(), nodeListModifiedKeys.item(i).getTextContent());
}
}
logger.debug("Map is " + toReturn);
namespacesCache.insert(resourceName, toReturn);
return toReturn;
} catch (Exception e) {
logger.error("Error while trying to fetch applicationProfile profile from the infrastructure", e);
return null;
}
}
}
/**
* Replace the extras' keys if needed
* @param customFields
@ -542,11 +553,11 @@ public abstract class HelperMethods {
*/
public static Map<String, List<String>> replaceFieldsKey(Map<String, List<String>> customFields,
Map<String, String> namespaces) {
Map<String, List<String>> toReturn = new HashMap<String, List<String>>();
Iterator<Entry<String, List<String>>> iterator = customFields.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<java.lang.String, java.util.List<java.lang.String>> entry = (Map.Entry<java.lang.String, java.util.List<java.lang.String>>) iterator
.next();
@ -555,7 +566,7 @@ public abstract class HelperMethods {
else
toReturn.put(entry.getKey(), entry.getValue());
}
return toReturn;
}
}
}

30
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/AssociationToGroupThread.java
View File

@ -27,21 +27,25 @@ public class AssociationToGroupThread extends Thread {
private String username;
private DataCatalogue catalogue;
private String organizationId;
private String apiKey;
/**
* @param groupTitle
*
* @param groupsTitles
* @param datasetId
* @param username
* @param organizationId
* @param username
* @param catalogue
* @param apiKey
*/
public AssociationToGroupThread(List<String> groupsTitles, String datasetId, String organizationId,
String username, DataCatalogue catalogue) {
String username, DataCatalogue catalogue, String apiKey) {
this.groupsTitles = groupsTitles;
this.datasetId = datasetId;
this.username = username;
this.catalogue = catalogue;
this.organizationId = organizationId;
this.apiKey = apiKey;
}
@Override
@ -52,16 +56,14 @@ public class AssociationToGroupThread extends Thread {
logger.info("Association thread started to put the dataset with id="+ datasetId + " into group with title(s) " + groupsTitles + " for user " + username);
// find parents' groups
String userApiKey = catalogue.getApiKeyFromUsername(username);
findHierarchy(groupsTitles, catalogue, userApiKey);
findHierarchy(groupsTitles, catalogue, apiKey);
Set<String> uniqueGroups = new HashSet<String>(groupsTitles);
logger.info("Full set of groups is " + uniqueGroups);
// retrieve the role to be assigned according the one the user has into the organization of the dataset
RolesCkanGroupOrOrg role = RolesCkanGroupOrOrg.valueOf(catalogue.getRoleOfUserInOrganization(username, organizationId, userApiKey).toUpperCase());
RolesCkanGroupOrOrg role = RolesCkanGroupOrOrg.valueOf(catalogue.getRoleOfUserInOrganization(username, organizationId, apiKey).toUpperCase());
if(!role.equals(RolesCkanGroupOrOrg.ADMIN))
role = RolesCkanGroupOrOrg.MEMBER; // decrease the role to member if it is not an admin
@ -76,7 +78,7 @@ public class AssociationToGroupThread extends Thread {
continue;
}
else{
boolean putIntoGroup = catalogue.assignDatasetToGroup(groupTitle, datasetId, userApiKey);
boolean putIntoGroup = catalogue.assignDatasetToGroup(groupTitle, datasetId, apiKey);
logger.info("Was product put into group " + groupTitle + "? " + putIntoGroup);
}
}
@ -98,10 +100,10 @@ public class AssociationToGroupThread extends Thread {
String apiKey) {
ListIterator<String> iterator = groupsTitles.listIterator();
while (iterator.hasNext()) {
String group = (String) iterator.next();
List<CkanGroup> parents = catalogue.getParentGroups(group, apiKey);
if(parents == null || parents.isEmpty())
@ -110,13 +112,13 @@ public class AssociationToGroupThread extends Thread {
for (CkanGroup ckanGroup : parents) {
List<String> parentsList = new ArrayList<String>(Arrays.asList(ckanGroup.getName()));
findHierarchy(parentsList, catalogue, apiKey);
for (String parent : parentsList) {
iterator.add(parent);
}
}
}
}
}
}

11
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java
View File

@ -41,7 +41,6 @@ import eu.trentorise.opendata.jackan.model.CkanResourceBase;
*/
public class ManageTimeSeriesThread extends Thread{
private static final String CSV_FILE_FORMAT = ".csv";
private static final String PATH_SEPARATOR = "/";
// Logger
@ -50,7 +49,7 @@ public class ManageTimeSeriesThread extends Thread{
// try to attach the source at most CANCHES times ..
private static final int CANCHES = 3;
private static CacheInterface<String, WorkspaceCatalogue> vreFolderCache = new CacheImpl<String, WorkspaceCatalogue>(1000 * 60 * 120);
private static CacheInterface<String, WorkspaceCatalogue> vreFolderCache = new CacheImpl<String, WorkspaceCatalogue>(1000 * 60 * 60 * 24);
private Common record;
private String uuidKB;
@ -164,7 +163,7 @@ public class ManageTimeSeriesThread extends Thread{
char firstLetter = uuidKB.charAt(0);
// the whole path of the directory is going to be...
String csvDirectoryForThisProduct = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + replaceIllegalChars(uuidKB) + PATH_SEPARATOR + "csv";
String csvDirectoryForThisProduct = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + replaceIllegalChars(uuidKB) + PATH_SEPARATOR + CSVHelpers.CSV_EXTENSION.replace(".", "");
logger.debug("The path under which the time series are going to be saved is " + csvDirectoryForThisProduct);
WorkspaceFolder csvFolder = HelperMethods.createOrGetSubFoldersByPath(catalogueFolder, csvDirectoryForThisProduct);
@ -187,7 +186,7 @@ public class ManageTimeSeriesThread extends Thread{
CustomField customAnnotation = field.getAnnotation(CustomField.class);
logger.debug("A time series has been just found (from field " + customAnnotation.key() + ")");
String resourceToAttachOnCkanName = (replaceIllegalChars(productName) + "_" + customAnnotation.key()).replaceAll("\\s", "_").replaceAll("[_]+", "_") + CSV_FILE_FORMAT;
String resourceToAttachOnCkanName = (replaceIllegalChars(productName) + "_" + customAnnotation.key()).replaceAll("\\s", "_").replaceAll("[_]+", "_") + CSVHelpers.CSV_EXTENSION;
String resourceToAttachOnCkanDescription = productName + " : " + customAnnotation.key() + " time series";
File csvFile = CSVHelpers.listToCSV(asList);
@ -206,7 +205,7 @@ public class ManageTimeSeriesThread extends Thread{
if(ckanResource != null){
if(createdFileOnWorkspace == null)
createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, uuidKB + CSV_FILE_FORMAT, resourceToAttachOnCkanDescription, csvFile);
createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, uuidKB + CSVHelpers.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile);
if(createdFileOnWorkspace != null){
@ -265,4 +264,4 @@ public class ManageTimeSeriesThread extends Thread{
resourceToAttachName,
description);
}
}
}

2
src/main/webapp/WEB-INF/web.xml
View File

@ -36,7 +36,7 @@
<description>Context of products no longer under manage activities</description>
<param-name>PublicVRE</param-name>
<!-- <param-value>/d4science.research-infrastructures.eu/FARM/GRSF</param-value> -->
<param-value>/gcube/devNext/NextNext</param-value>
<param-value>/gcube/devsec/devVRE</param-value>
<!-- <param-value>/gcube/preprod/preVRE</param-value> -->
</context-param>

2
src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java
View File

@ -475,7 +475,7 @@ public class JTests {
public void testAssociationThread() throws InterruptedException{
String name = "low-abundance";
DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance("/gcube/devNext/NextNext");
AssociationToGroupThread threadGroups = new AssociationToGroupThread(Arrays.asList(name), "another-test-test-please-ignore", "grsf", "costantino_perciante", catalogue);
AssociationToGroupThread threadGroups = new AssociationToGroupThread(Arrays.asList(name), "another-test-test-please-ignore", "grsf", "costantino_perciante", catalogue, "apiKey");
threadGroups.start();
threadGroups.join();
logger.info("Thread stopped!");