2018-12-04 12:06:22 +01:00
package org.gcube.gcat.persistence.ckan ;
import java.util.ArrayList ;
import java.util.HashMap ;
import java.util.List ;
import java.util.Map ;
import javax.ws.rs.BadRequestException ;
import javax.ws.rs.DELETE ;
import javax.ws.rs.GET ;
import javax.ws.rs.HEAD ;
import javax.ws.rs.InternalServerErrorException ;
import javax.ws.rs.NotAllowedException ;
import javax.ws.rs.OPTIONS ;
import javax.ws.rs.PUT ;
import javax.ws.rs.WebApplicationException ;
2019-05-20 17:23:49 +02:00
import javax.ws.rs.core.MultivaluedMap ;
2018-12-04 12:06:22 +01:00
import javax.ws.rs.core.Response.Status ;
import org.gcube.common.scope.impl.ScopeBean ;
import org.gcube.common.scope.impl.ScopeBean.Type ;
import org.gcube.gcat.annotation.PURGE ;
2019-05-20 17:23:49 +02:00
import org.gcube.gcat.api.GCatConstants ;
2018-12-04 12:06:22 +01:00
import org.gcube.gcat.oldutils.Validator ;
2019-01-10 12:29:47 +01:00
import org.gcube.gcat.profile.MetadataUtility ;
2019-09-12 14:26:24 +02:00
import org.gcube.gcat.social.SocialPost ;
2018-12-04 12:06:22 +01:00
import org.gcube.gcat.utils.ContextUtility ;
import org.gcube.gcat.utils.URIResolver ;
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
import com.fasterxml.jackson.databind.JsonNode ;
import com.fasterxml.jackson.databind.node.ArrayNode ;
import com.fasterxml.jackson.databind.node.ObjectNode ;
/ * *
* @author Luca Frosini ( ISTI - CNR )
* /
public class CKANPackage extends CKAN {
private static final Logger logger = LoggerFactory . getLogger ( CKANPackage . class ) ;
2019-02-08 12:25:19 +01:00
/ *
2018-12-04 12:06:22 +01:00
// see https://docs.ckan.org/en/latest/api/#ckan.logic.action.get.package_list
public static final String ITEM_LIST = CKAN . CKAN_API_PATH + " package_list " ;
2019-02-08 12:25:19 +01:00
* /
// see https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search
public static final String ITEM_LIST = CKAN . CKAN_API_PATH + " package_search " ;
2018-12-04 12:06:22 +01:00
// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.create.package_create
public static final String ITEM_CREATE = CKAN . CKAN_API_PATH + " package_create " ;
// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.get.package_show
public static final String ITEM_SHOW = CKAN . CKAN_API_PATH + " package_show " ;
// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.update.package_update
public static final String ITEM_UPDATE = CKAN . CKAN_API_PATH + " package_update " ;
// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.patch.package_patch
public static final String ITEM_PATCH = CKAN . CKAN_API_PATH + " package_patch " ;
// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.delete.package_delete
public static final String ITEM_DELETE = CKAN . CKAN_API_PATH + " package_delete " ;
// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.delete.dataset_purge
public static final String ITEM_PURGE = CKAN . CKAN_API_PATH + " dataset_purge " ;
2019-02-08 12:25:19 +01:00
// limit in https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search
protected static final String ROWS_KEY = " rows " ;
// offset in https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search
protected static final String START_KEY = " start " ;
protected static final String Q_KEY = " q " ;
protected static final String ORGANIZATION_FILTER_TEMPLATE = " organization:%s " ;
2018-12-04 12:06:22 +01:00
protected static final String LICENSE_KEY = " license_id " ;
2019-02-26 14:58:06 +01:00
protected static final String EXTRAS_ITEM_URL_KEY = " Item URL " ;
2018-12-04 12:06:22 +01:00
protected static final String AUTHOR_KEY = " author " ;
protected static final String AUTHOR_EMAIL_KEY = " author_email " ;
protected static final String OWNER_ORG_KEY = " owner_org " ;
protected static final String RESOURCES_KEY = " resources " ;
protected static final String TITLE_KEY = " title " ;
2019-02-08 12:04:20 +01:00
public static final String EXTRAS_KEY = " extras " ;
public static final String EXTRAS_KEY_KEY = " key " ;
public static final String EXTRAS_KEY_VALUE_SYSTEM_TYPE = " system:type " ;
public static final String EXTRAS_VALUE_KEY = " value " ;
2019-02-08 14:40:57 +01:00
// The 'results' array is included in the 'result' object for package_search
private static final String RESULTS_KEY = " results " ;
2019-02-08 16:49:30 +01:00
protected static final String PRIVATE_KEY = " private " ;
2019-02-08 14:40:57 +01:00
protected static final String SEARCHABLE_KEY = " searchable " ;
2019-05-20 17:23:49 +02:00
protected static final String CAPACITY_KEY = " capacity " ;
2019-02-08 16:49:30 +01:00
2019-02-08 15:17:09 +01:00
// protected static final String INCLUDE_PRIVATE_KEY = "include_private";
2019-02-08 14:40:57 +01:00
// protected static final String INCLUDE_DRAFTS_KEY = "include_drafts";
2018-12-04 12:06:22 +01:00
public static final String GROUPS_KEY = " groups " ;
public static final String TAGS_KEY = " tags " ;
2019-09-12 14:26:24 +02:00
2018-12-04 12:06:22 +01:00
protected final List < CKANResource > managedResources ;
protected String itemID ;
public CKANPackage ( ) {
super ( ) ;
LIST = ITEM_LIST ;
CREATE = ITEM_CREATE ;
READ = ITEM_SHOW ;
UPDATE = ITEM_UPDATE ;
PATCH = ITEM_PATCH ;
DELETE = ITEM_DELETE ;
PURGE = ITEM_PURGE ;
managedResources = new ArrayList < CKANResource > ( ) ;
}
2019-02-08 12:25:19 +01:00
/ *
* Return the CKAN organization name using the current context name
* /
protected String getOrganizationName ( ScopeBean scopeBean ) {
String contextName = scopeBean . name ( ) ;
return contextName . toLowerCase ( ) . replace ( " " , " _ " ) ;
}
protected String getOrganizationName ( ) {
ScopeBean scopeBean = new ScopeBean ( ContextUtility . getCurrentContext ( ) ) ;
return getOrganizationName ( scopeBean ) ;
}
2018-12-04 12:06:22 +01:00
public ObjectNode checkBaseInformation ( String json ) throws Exception {
ObjectNode objectNode = ( ObjectNode ) mapper . readTree ( json ) ;
objectNode = ( ObjectNode ) checkName ( objectNode ) ;
// We need to enforce the itemID to properly manage resource persistence
if ( objectNode . has ( ID_KEY ) ) {
itemID = objectNode . get ( ID_KEY ) . asText ( ) ;
}
2019-02-08 16:49:30 +01:00
// To include private item in search result (e.g. listing) a private package must be searchable
// The package it is jsut included in the search and listing results but remain private and cannot be accessed
// if not authorized
if ( objectNode . has ( PRIVATE_KEY ) ) {
boolean privatePackage = objectNode . get ( PRIVATE_KEY ) . asBoolean ( ) ;
if ( privatePackage ) {
objectNode . put ( SEARCHABLE_KEY , true ) ;
}
}
2018-12-04 12:06:22 +01:00
// check license
String licenseId = null ;
if ( objectNode . has ( LICENSE_KEY ) ) {
licenseId = objectNode . get ( LICENSE_KEY ) . asText ( ) ;
}
if ( licenseId = = null | | licenseId . isEmpty ( ) ) {
throw new BadRequestException (
" You must specify a license identifier for the item. License list can be retrieved using licence collection " ) ;
2019-05-20 17:23:49 +02:00
} else {
try {
CKANLicense . checkLicenseId ( licenseId ) ;
} catch ( Exception e ) {
throw new BadRequestException (
" You must specify an existing license identifier for the item. License list can be retrieved using licence collection " ) ;
}
2018-12-04 12:06:22 +01:00
}
2019-05-20 17:23:49 +02:00
if ( objectNode . has ( CAPACITY_KEY ) ) {
/ *
* When a client provides the ' capacity ' field as ' private ' , the item is not counted in the
* total number of items in the GUI . We want to avoid such a behavior
* See https : //support.d4science.org/issues/16410
* /
objectNode . remove ( CAPACITY_KEY ) ;
}
2019-09-12 14:26:24 +02:00
2019-09-16 12:00:49 +02:00
CKANUser ckanUser = CKANUserCache . getCurrrentCKANUser ( ) ;
2019-09-12 14:26:24 +02:00
objectNode . put ( AUTHOR_KEY , ckanUser . getName ( ) ) ;
objectNode . put ( AUTHOR_EMAIL_KEY , ckanUser . getPortalUser ( ) . getEMail ( ) ) ;
2018-12-04 12:06:22 +01:00
// owner organization must be specified if the token belongs to a VRE
ScopeBean scopeBean = new ScopeBean ( ContextUtility . getCurrentContext ( ) ) ;
String gotOrganization = null ;
if ( objectNode . has ( OWNER_ORG_KEY ) ) {
gotOrganization = objectNode . get ( OWNER_ORG_KEY ) . asText ( ) ;
}
if ( scopeBean . is ( Type . VRE ) ) {
2019-02-08 12:25:19 +01:00
String organizationFromContext = getOrganizationName ( scopeBean ) ;
2018-12-04 12:06:22 +01:00
if ( gotOrganization ! = null ) {
if ( gotOrganization . compareTo ( organizationFromContext ) ! = 0 ) {
CKANOrganization ckanOrganization = new CKANOrganization ( ) ;
ckanOrganization . setName ( organizationFromContext ) ;
ckanOrganization . read ( ) ;
String organizationID = null ;
if ( ckanOrganization . result . has ( ID_KEY ) ) {
organizationID = ckanOrganization . result . get ( ID_KEY ) . asText ( ) ;
}
if ( organizationID = = null | | gotOrganization . compareTo ( organizationID ) ! = 0 ) {
throw new BadRequestException (
" You can only publish in the Organization associate to the current VRE " ) ;
}
}
} else {
2019-05-20 17:23:49 +02:00
gotOrganization = organizationFromContext ;
2018-12-04 12:06:22 +01:00
objectNode . put ( OWNER_ORG_KEY , organizationFromContext ) ;
}
} else {
if ( gotOrganization = = null ) {
throw new BadRequestException ( " You must specify an Organization usign " + OWNER_ORG_KEY + " field " ) ;
}
}
2019-09-12 14:26:24 +02:00
ckanUser . addUserToOrganization ( gotOrganization ) ;
2019-05-20 17:23:49 +02:00
2018-12-04 12:06:22 +01:00
return objectNode ;
}
protected JsonNode validateJson ( String json ) {
try {
// check base information (and set them if needed)
ObjectNode objectNode = checkBaseInformation ( json ) ;
// Validating against profiles if any
2019-05-20 17:23:49 +02:00
MetadataUtility metadataUtility = new MetadataUtility ( ) ;
2019-01-10 12:29:47 +01:00
if ( ! metadataUtility . getMetadataProfiles ( ) . isEmpty ( ) ) {
2019-02-27 17:36:09 +01:00
Validator validator = new Validator ( mapper ) ;
2019-05-20 17:23:49 +02:00
objectNode = validator . validateAgainstProfile ( objectNode , metadataUtility ) ;
2018-12-04 12:06:22 +01:00
}
return objectNode ;
} catch ( BadRequestException e ) {
throw e ;
} catch ( Exception e ) {
throw new BadRequestException ( e ) ;
}
}
@Override
2019-01-10 12:29:47 +01:00
public String list ( int limit , int offset ) {
2019-02-08 12:25:19 +01:00
Map < String , String > parameters = new HashMap < > ( ) ;
if ( limit < = 0 ) {
// According to CKAN documentation
// the number of matching rows to return. There is a hard limit of 1000 datasets per query.
// see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search
limit = 1000 ;
}
parameters . put ( ROWS_KEY , String . valueOf ( limit ) ) ;
if ( offset < 0 ) {
offset = 0 ;
}
2019-02-08 14:40:57 +01:00
parameters . put ( START_KEY , String . valueOf ( offset * limit ) ) ;
2019-02-08 12:25:19 +01:00
String organizationName = getOrganizationName ( ) ;
String organizationQueryString = String . format ( ORGANIZATION_FILTER_TEMPLATE , organizationName ) ;
parameters . put ( Q_KEY , organizationQueryString ) ;
2019-02-08 15:17:09 +01:00
// parameters.put(INCLUDE_PRIVATE_KEY, String.valueOf(true));
2019-02-08 14:40:57 +01:00
// By default not including draft
// parameters.put(INCLUDE_DRAFTS_KEY, String.valueOf(false));
sendGetRequest ( LIST , parameters ) ;
ArrayNode results = ( ArrayNode ) result . get ( RESULTS_KEY ) ;
ArrayNode arrayNode = mapper . createArrayNode ( ) ;
for ( JsonNode node : results ) {
try {
2019-02-08 16:49:30 +01:00
String name = node . get ( NAME_KEY ) . asText ( ) ;
arrayNode . add ( name ) ;
2019-02-08 14:40:57 +01:00
} catch ( Exception e ) {
try {
logger . error ( " Unable to get the ID of {}. the result will not be included in the result " , mapper . writeValueAsString ( node ) ) ;
} catch ( Exception ex ) {
logger . error ( " " , ex ) ;
}
}
}
return getAsString ( arrayNode ) ;
2018-12-04 12:06:22 +01:00
}
protected void rollbackManagedResources ( ) {
for ( CKANResource ckanResource : managedResources ) {
try {
ckanResource . rollback ( ) ;
} catch ( Exception e ) {
logger . error ( " Unable to rollback resource {} to the original value " , ckanResource . getResourceID ( ) ) ;
}
}
}
protected ArrayNode createResources ( ArrayNode resourcesToBeCreated ) {
ArrayNode created = mapper . createArrayNode ( ) ;
for ( JsonNode resourceNode : resourcesToBeCreated ) {
CKANResource ckanResource = new CKANResource ( itemID ) ;
String json = ckanResource . create ( getAsString ( resourceNode ) ) ;
created . add ( getAsJsonNode ( json ) ) ;
managedResources . add ( ckanResource ) ;
}
return created ;
}
2019-02-26 14:58:06 +01:00
protected JsonNode addExtraField ( JsonNode jsonNode , String key , String value ) {
ArrayNode extras = null ;
boolean found = false ;
if ( jsonNode . has ( EXTRAS_KEY ) ) {
extras = ( ArrayNode ) jsonNode . get ( EXTRAS_KEY ) ;
for ( JsonNode extra : extras ) {
if ( extra . has ( EXTRAS_KEY_KEY ) & & extra . get ( EXTRAS_KEY_KEY ) . asText ( ) . compareTo ( key ) = = 0 ) {
( ( ObjectNode ) extra ) . put ( EXTRAS_VALUE_KEY , value ) ;
found = true ;
break ;
}
}
} else {
extras = mapper . createArrayNode ( ) ;
}
if ( ! found ) {
ObjectNode extra = mapper . createObjectNode ( ) ;
extra . put ( EXTRAS_KEY_KEY , key ) ;
extra . put ( EXTRAS_VALUE_KEY , value ) ;
extras . add ( extra ) ;
}
return jsonNode ;
}
protected String addItemURLViaResolver ( JsonNode jsonNode ) {
// Adding Item URL via Resolver
URIResolver uriResolver = new URIResolver ( ) ;
String catalogueItemURL = uriResolver . getCatalogueItemURL ( name ) ;
addExtraField ( jsonNode , EXTRAS_ITEM_URL_KEY , catalogueItemURL ) ;
return catalogueItemURL ;
}
2019-05-20 17:23:49 +02:00
protected void sendSocialPost ( String title , String catalogueItemURL ) {
try {
boolean socialPost = true ;
try {
MultivaluedMap < String , String > queryParameters = uriInfo . getQueryParameters ( ) ;
if ( queryParameters . containsKey ( GCatConstants . SOCIAL_POST_PARAMETER ) ) {
socialPost = Boolean . parseBoolean ( queryParameters . getFirst ( GCatConstants . SOCIAL_POST_PARAMETER ) ) ;
}
} catch ( Exception e ) {
socialPost = true ;
}
if ( socialPost ) {
ArrayNode arrayNode = ( ArrayNode ) result . get ( TAGS_KEY ) ;
2019-09-12 14:26:24 +02:00
SocialPost socialService = new SocialPost ( ) ;
2019-05-20 17:23:49 +02:00
socialService . setItemID ( itemID ) ;
socialService . setItemURL ( catalogueItemURL ) ;
socialService . setTags ( arrayNode ) ;
socialService . setItemTitle ( title ) ;
socialService . start ( ) ;
} else {
logger . info ( " The request explicitly disabled the Social Post. " ) ;
}
} catch ( Exception e ) {
logger . warn ( " error dealing with Social Post. The service will not raise the exception belove. Please contact the administrator to let him know about this message. " , e ) ;
}
}
2018-12-04 12:06:22 +01:00
// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.create.package_create
@Override
public String create ( String json ) {
try {
logger . debug ( " Going to create Item {} " , json ) ;
JsonNode jsonNode = validateJson ( json ) ;
ArrayNode resourcesToBeCreated = mapper . createArrayNode ( ) ;
if ( jsonNode . has ( RESOURCES_KEY ) ) {
resourcesToBeCreated = ( ArrayNode ) jsonNode . get ( RESOURCES_KEY ) ;
( ( ObjectNode ) jsonNode ) . remove ( RESOURCES_KEY ) ;
}
2019-05-20 17:23:49 +02:00
ScopeBean scopeBean = new ScopeBean ( ContextUtility . getCurrentContext ( ) ) ;
String catalogueItemURL = " " ;
if ( scopeBean . is ( Type . VRE ) ) {
catalogueItemURL = addItemURLViaResolver ( jsonNode ) ;
}
2019-02-26 14:58:06 +01:00
2018-12-04 12:06:22 +01:00
super . create ( getAsString ( jsonNode ) ) ;
this . itemID = result . get ( ID_KEY ) . asText ( ) ;
ArrayNode created = createResources ( resourcesToBeCreated ) ;
( ( ObjectNode ) result ) . replace ( RESOURCES_KEY , created ) ;
2019-02-26 14:58:06 +01:00
// Adding Item URL via Resolver as
// ((ObjectNode) result).put(ITEM_URL_KEY, catalogueItemURL);
2018-12-04 12:06:22 +01:00
// Actions performed after a package has been correctly created on ckan.
String title = result . get ( TITLE_KEY ) . asText ( ) ;
2019-05-20 17:23:49 +02:00
if ( scopeBean . is ( Type . VRE ) ) {
sendSocialPost ( title , catalogueItemURL ) ;
}
2018-12-04 12:06:22 +01:00
return getAsString ( result ) ;
} catch ( WebApplicationException e ) {
rollbackManagedResources ( ) ;
throw e ;
} catch ( Exception e ) {
rollbackManagedResources ( ) ;
throw new InternalServerErrorException ( e ) ;
}
}
// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.update.package_update
@Override
public String update ( String json ) {
try {
JsonNode jsonNode = validateJson ( json ) ;
read ( ) ;
2019-07-22 11:54:05 +02:00
this . itemID = result . get ( ID_KEY ) . asText ( ) ;
2018-12-04 12:06:22 +01:00
Map < String , CKANResource > originalResources = new HashMap < > ( ) ;
ArrayNode originalResourcesarrayNode = ( ArrayNode ) result . get ( RESOURCES_KEY ) ;
if ( originalResources ! = null ) {
for ( JsonNode resourceNode : originalResourcesarrayNode ) {
CKANResource ckanResource = new CKANResource ( itemID ) ;
ckanResource . setPreviousRepresentation ( resourceNode ) ;
String resourceID = ckanResource . getResourceID ( ) ;
originalResources . put ( resourceID , ckanResource ) ;
}
}
if ( jsonNode . has ( RESOURCES_KEY ) ) {
ArrayNode resourcesToBeSend = mapper . createArrayNode ( ) ;
ArrayNode receivedResources = ( ArrayNode ) jsonNode . get ( RESOURCES_KEY ) ;
for ( JsonNode resourceNode : receivedResources ) {
CKANResource ckanResource = new CKANResource ( itemID ) ;
String resourceId = CKANResource . extractResourceID ( resourceNode ) ;
if ( resourceId ! = null ) {
if ( originalResources . containsKey ( resourceId ) ) {
ckanResource = originalResources . get ( resourceId ) ;
originalResources . remove ( resourceId ) ;
} else {
throw new BadRequestException ( " The content cotains a resource with id " + resourceId + " which does not exists " ) ;
}
}
resourceNode = ckanResource . createOrUpdate ( resourceNode ) ;
resourcesToBeSend . add ( resourceNode ) ;
managedResources . add ( ckanResource ) ;
}
( ( ObjectNode ) jsonNode ) . replace ( RESOURCES_KEY , resourcesToBeSend ) ;
}
2019-02-26 14:58:06 +01:00
addItemURLViaResolver ( jsonNode ) ;
2018-12-04 12:06:22 +01:00
sendPostRequest ( ITEM_UPDATE , getAsString ( jsonNode ) ) ;
for ( String resourceId : originalResources . keySet ( ) ) {
CKANResource ckanResource = originalResources . get ( resourceId ) ;
ckanResource . deleteFile ( ) ;
}
2019-02-26 14:58:06 +01:00
/ *
2018-12-04 12:06:22 +01:00
// Adding Item URL via Resolver
URIResolver uriResolver = new URIResolver ( ) ;
String catalogueItemURL = uriResolver . getCatalogueItemURL ( name ) ;
( ( ObjectNode ) result ) . put ( ITEM_URL_KEY , catalogueItemURL ) ;
2019-02-26 14:58:06 +01:00
* /
2018-12-04 12:06:22 +01:00
return getAsString ( result ) ;
} catch ( WebApplicationException e ) {
rollbackManagedResources ( ) ;
throw e ;
} catch ( Exception e ) {
rollbackManagedResources ( ) ;
throw new InternalServerErrorException ( e ) ;
}
}
// see http://docs.ckan.org/en/latest/api/#ckan.logic.action.patch.package_patch
@Override
public String patch ( String json ) {
String [ ] moreAllowed = new String [ ] { HEAD . class . getSimpleName ( ) , GET . class . getSimpleName ( ) ,
PUT . class . getSimpleName ( ) , DELETE . class . getSimpleName ( ) , PURGE . class . getSimpleName ( ) } ;
throw new NotAllowedException ( OPTIONS . class . getSimpleName ( ) , moreAllowed ) ;
}
@Override
protected void delete ( ) {
super . delete ( ) ;
}
@Override
public void purge ( ) {
try {
delete ( ) ;
} catch ( WebApplicationException e ) {
// If the item was deleted but not purged we obtain Not Found. This is accepted. The item has to be purged
// with SysAdmin right.
Status status = Status . fromStatusCode ( e . getResponse ( ) . getStatusInfo ( ) . getStatusCode ( ) ) ;
if ( status ! = Status . NOT_FOUND ) {
throw e ;
}
}
setApiKey ( CKANUtility . getSysAdminAPI ( ) ) ;
read ( ) ;
if ( result . has ( RESOURCES_KEY ) ) {
itemID = result . get ( ID_KEY ) . asText ( ) ;
ArrayNode arrayNode = ( ArrayNode ) result . get ( RESOURCES_KEY ) ;
for ( JsonNode jsonNode : arrayNode ) {
CKANResource ckanResource = new CKANResource ( itemID ) ;
ckanResource . setPreviousRepresentation ( jsonNode ) ;
2019-01-10 12:29:47 +01:00
ckanResource . deleteFile ( ) ; // Only delete file is required because the item has been deleted
2018-12-04 12:06:22 +01:00
}
}
super . purge ( ) ;
}
}