ckanext-d4science_theme/ckanext/d4science_theme/plugin.py

# encoding: utf-8
from logging import getLogger

import ckan.plugins as plugins
from ckanext.d4science_theme import helpers
import ckan.plugins.toolkit as toolkit
import ckan.lib.dictization.model_save as model_save
import ckan.model as model
import ckan.lib.helpers as h
import sqlalchemy as sa
from ckan.controllers.home import HomeController
from ckanext.d4science_theme.controllers.organization import OrganizationVREController
from ckanext.d4science_theme.controllers.home import d4SHomeController
from ckan.config.middleware.common_middleware import TrackingMiddleware
from ckan.common import session
from ckan.plugins import IRoutes

from ckan.common import (
   g
)

# Created by Francesco Mangiacrapa
# francesco.mangiacrapa@isti.cnr.it 
# ISTI-CNR Pisa (ITALY)

log = getLogger(__name__)

d4s_ctg_namespaces_controller = None

def remove_check_replicated_custom_key(schema):
    if schema is not None:
        schema.pop('__before', None)
    
    return schema

#CREATED BY FRANCESCO MANGIACRAPA FOR OVERRIDING THE package_extras_save FROM dictization.model_save.py
def _package_extras_save(extra_dicts, obj, context):
    ''' It can save repeated extras as key-value '''
    allow_partial_update = context.get("allow_partial_update", False)
    if extra_dicts is None and allow_partial_update:
        return
     
    model = context["model"]
    session = context["session"]
     
    #ADDED BY FRANCESCO MANGIACRAPA
    log.debug("extra_dicts: "+unicode(str(extra_dicts)).encode('utf-8'))
    #print "extra_dicts: "+str(extra_dicts)
     
    extras_list = obj.extras_list
    #extras = dict((extra.key, extra) for extra in extras_list)
    old_extras = {}
    extras = {}
    for extra in extras_list or []:
        old_extras.setdefault(extra.key, []).append(extra.value)
        extras.setdefault(extra.key, []).append(extra)
    
    #ADDED BY FRANCESCO MANGIACRAPA       
    #print "old_extras: "+str(old_extras)
    
    new_extras = {}
    for extra_dict in extra_dicts or []:
        #print 'extra_dict key: '+extra_dict["key"] + ', value: '+extra_dict["value"]
        #new_extras.setdefault(extra_dict["key"], []).append(extra_dict["value"])
        if extra_dict.get("deleted"):
            log.debug("extra_dict deleted: "+unicode(extra_dict["key"]).encode('utf-8'))
            #print 'extra_dict deleted: '+extra_dict["key"]
            continue
        
        #if extra_dict['value'] is not None and not extra_dict["value"] == "":
        if extra_dict['value'] is not None:
            new_extras.setdefault(extra_dict["key"], []).append(extra_dict["value"])
    
    #ADDED BY FRANCESCO MANGIACRAPA
    log.debug("new_extras: "+unicode(str(new_extras)).encode('utf-8'))
    #print "new_extras: "+str(new_extras)
    
    #new
    for key in set(new_extras.keys()) - set(old_extras.keys()):
        state = 'active'
        log.debug("adding key: "+unicode(key).encode('utf-8'))
        #print "adding key: "+str(key)
        extra_lst = new_extras[key]
        for extra in extra_lst:
            extra = model.PackageExtra(state=state, key=key, value=extra)
            session.add(extra)
            extras_list.append(extra)
    
    #deleted
    for key in set(old_extras.keys()) - set(new_extras.keys()):
        log.debug("deleting key: "+unicode(key).encode('utf-8'))
        #print "deleting key: "+str(key)
        extra_lst = extras[key]
        for extra in extra_lst:
            state = 'deleted'
            extra.state = state
            extras_list.remove(extra)
            
    #changed
    for key in set(new_extras.keys()) & set(old_extras.keys()):
        #for each value of new list
        for value in new_extras[key]:
            old_occur = old_extras[key].count(value)
            new_occur = new_extras[key].count(value)
            log.debug("value: "+unicode(value).encode('utf-8') + ", new_occur: "+unicode(new_occur).encode('utf-8')+ ", old_occur: "+unicode(old_occur).encode('utf-8'))
            #print "value: "+str(value)  + ", new_occur: "+str(new_occur) + ", old_occur: "+str(old_occur)
            # it is an old value deleted or not
            if value in old_extras[key]:
                if old_occur == new_occur:
                    #print "extra - occurrences of: "+str(value)  +", are equal into both list"
                    log.debug("extra - occurrences of: "+unicode(value).encode('utf-8') +", are equal into both list")
                    #there is a little bug, this code return always the first element, so I'm fixing with #FIX-STATUS
                    extra_values = get_package_for_value(extras[key], value)
                    #extras_list.append(extra)
                    for extra in extra_values:
                        state = 'active'
                        extra.state = state
                        session.add(extra)
                        #print "extra updated: "+str(extra)
                        log.debug("extra updated: "+unicode(extra).encode('utf-8'))
                
                elif new_occur > old_occur:
                    #print "extra - a new occurrence of: "+str(value)  +", is present into new list, adding it to old list"
                    log.debug("extra - a new occurrence of: "+unicode(value).encode('utf-8') +", is present into new list, adding it to old list")
                    state = 'active'
                    extra = model.PackageExtra(state=state, key=key, value=value)
                    extra.state = state
                    session.add(extra)
                    extras_list.append(extra)
                    old_extras[key].append(value)
                    log.debug("old extra values updated: "+unicode(old_extras[key]).encode('utf-8'))
                    #print "old extra values updated: "+str(old_extras[key])
                
                else:
                    #remove all occurrences deleted - this code could be optimized, it is run several times but could be performed one shot
                    countDelete = old_occur-new_occur
                    log.debug("extra - occurrence of: "+unicode(value).encode('utf-8')+", is not present into new list, removing "+unicode(countDelete).encode('utf-8')+" occurrence/s from old list")
                    #print "extra - occurrence of: "+str(value)  +", is not present into new list, removing "+str(countDelete)+" occurrence/s from old list"
                    extra_values = get_package_for_value(extras[key], value)
                    for idx, extra in enumerate(extra_values):
                        if idx < countDelete:
                            #print "extra - occurrence of: "+str(value)  +", is not present into new list, removing it from old list"
                            log.debug("pkg extra deleting: "+unicode(extra.value).encode('utf-8'))
                            #print "pkg extra deleting: "+str(extra.value)
                            state = 'deleted'
                            extra.state = state
                        
                        else:
                            #print "pkg extra reactivating: "+str(extra.value)
                            log.debug("pkg extra reactivating: "+unicode(extra.value).encode('utf-8'))
                            state = 'active'
                            extra.state = state
                            session.add(extra)
            
            else:
                #print "extra new value: "+str(value)
                log.debug("extra new value: "+unicode(value).encode('utf-8'))
                state = 'active'
                extra = model.PackageExtra(state=state, key=key, value=value)
                extra.state = state
                session.add(extra)
                extras_list.append(extra)
        
        
        #for each value of old list
        for value in old_extras[key]:
            #if value is not present in new list
            if value not in new_extras[key]:
                extra_values = get_package_for_value(extras[key], value)
                for extra in extra_values:
                    #print "not present extra deleting: "+str(extra)
                    log.debug("not present extra deleting: "+unicode(extra).encode('utf-8'))
                    state = 'deleted'
                    extra.state = state


#ADDED BY FRANCESCO MANGIACRAPA
def get_package_for_value(list_package, value):
    ''' Returns a list of packages containing the value passed in input
    '''
    lst = []
    for x in list_package:
        if x.value == value:
            lst.append(x)
    else:
        return lst
    
    return lst

#OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE
def _init_TrackingMiddleware(self, app, config):
    self.app = app
    log.debug('TrackingMiddleware d4Science instance')
    sqlalchemy_url = config.get('sqlalchemy.url')
    log.debug('sqlalchemy_url read: '+str(sqlalchemy_url))
    
    sqlalchemy_pool = config.get('sqlalchemy.pool_size')
    if sqlalchemy_pool is None:
        sqlalchemy_pool = 5
    
    log.debug('sqlalchemy_pool read: '+str(sqlalchemy_pool))
    sqlalchemy_overflow = config.get('sqlalchemy.max_overflow')
    
    if sqlalchemy_overflow is None:
        sqlalchemy_overflow = 10;
    
    log.debug('sqlalchemy_overflow read: '+str(sqlalchemy_overflow))
    
    try:
        self.engine = sa.create_engine(sqlalchemy_url, pool_size=int(sqlalchemy_pool), max_overflow=int(sqlalchemy_overflow))
    except TypeError as e:
        log.error('pool size does not work: ' +str(e.args))
        self.engine = sa.create_engine(sqlalchemy_url)
            


class D4Science_ThemePlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IDatasetForm)
    plugins.implements(plugins.ITemplateHelpers)
    plugins.implements(plugins.IFacets)
    plugins.implements(IRoutes, inherit=True)
    
    # IConfigurer
    def update_config(self, config_):
        # Add this plugin's templates dir to CKAN's extra_template_paths, so
        # that CKAN will use this plugin's custom templates.
        toolkit.add_template_directory(config_, 'templates')
        
        # Add this plugin's public dir to CKAN's extra_public_paths, so
        # that CKAN will use this plugin's custom static files.
        toolkit.add_public_directory(config_, 'public')
        
        # Register this plugin's fanstatic directory with CKAN.
        # Here, 'fanstatic' is the path to the fanstatic directory
        # (relative to this plugin.py file), and 'example_theme' is the name
        # that we'll use to refer to this fanstatic directory from CKAN
        # templates.
        toolkit.add_resource('fanstatic', 'd4science_theme')
    
    #IDatasetForm
    def create_package_schema(self):
        # let's grab the default schema in our plugin
        schema = super(D4Science_ThemePlugin, self).create_package_schema()
        schema = remove_check_replicated_custom_key(schema)
        #d.package_dict_save = _package_dict_save
        return schema
    
    #IDatasetForm
    def update_package_schema(self):
        schema = super(D4Science_ThemePlugin, self).update_package_schema()
        schema = remove_check_replicated_custom_key(schema)
        return schema
    
    #IDatasetForm
    def show_package_schema(self):
        schema = super(D4Science_ThemePlugin, self).show_package_schema()
        schema = remove_check_replicated_custom_key(schema)
        return schema
    
    #IDatasetForm
    def is_fallback(self):
        # Return True to register this plugin as the default handler for package types not handled by any other IDatasetForm plugin
        return False
    
    #IDatasetForm
    def package_types(self):
        # This plugin doesn't handle any special package types, it just
        # registers itself as the default (above).
        return []
    
    
    #ITemplateHelpers
    def get_helpers(self):
        log.info("get_helpers called...")
        '''Register functions as a template
        helper function.
        '''
        # Template helper function names should begin with the name of the
        # extension they belong to, to avoid clashing with functions from
        # other extensions.
        return {
        'd4science_theme_get_user_role_for_group_or_org': helpers.get_user_role_for_group_or_org,
        'd4science_theme_get_parents_for_group': helpers.get_parents_for_group,
        'get_header_param': helpers.get_header_param,
        'get_request_param': helpers.get_request_param,
        'get_cookie_value': helpers.get_cookie_value,
        'd4science_theme_markdown_extract_html' : helpers.markdown_extract_html,
        'd4science_theme_get_systemtype_value_from_extras' : helpers.get_systemtype_value_from_extras,
        'd4science_theme_get_systemtype_field_dict_from_session' : helpers.get_systemtype_field_dict_from_session,
        'd4science_theme_get_namespace_separator_from_session' : helpers.get_namespace_separator_from_session,
        'd4science_theme_get_extras' : helpers.get_extras,
        'd4science_theme_count_facet_items_dict' : helpers.count_facet_items_dict,
        'd4science_theme_purge_namespace_to_facet': helpers.purge_namespace_to_fieldname,
        'd4science_get_color_for_type': helpers.get_color_for_type,
        'd4science_get_d4s_namespace_controller': helpers.get_d4s_namespace_controller,
        'd4science_get_extras_indexed_for_namespaces': helpers.get_extras_indexed_for_namespaces,
        'd4science_get_namespaces_dict': helpers.get_namespaces_dict,
        'd4science_get_extra_for_category' : helpers.get_extra_for_category,
        'd4science_get_ordered_dictionary': helpers.ordered_dictionary,
        'd4science_get_qrcode_for_url': helpers.qrcode_for_url,
        'd4science_get_list_of_organizations': helpers.get_list_of_organizations,
        'd4science_get_image_display_for_group': helpers.get_image_display_for_group,
        'd4science_get_list_of_groups': helpers.get_list_of_groups,
        'd4science_get_browse_info_for_organisations_or_groups': helpers.get_browse_info_for_organisations_or_groups,
        'd4science_get_user_info': helpers.get_user_info,
        'd4science_get_url_to_icon_for_ckan_entity' : helpers.get_url_to_icon_for_ckan_entity,
        'd4science_get_ckan_translate_for' : helpers.get_ckan_translate_for,
        'd4science_get_location_to_bboxes' : helpers.get_location_to_bboxes,
        'd4science_get_content_moderator_system_placeholder': helpers.get_content_moderator_system_placeholder,
        }
    
    #Overriding package_extras_save method
    model_save.package_extras_save = _package_extras_save
    
    #Overriding index home controller
    d4sHC = d4SHomeController()
    HomeController.index = d4sHC.index
    
    #OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE
    TrackingMiddleware.__init__ = _init_TrackingMiddleware

    global d4s_ctg_namespaces_controller
    
    #if d4s_ctg_namespaces_controller is None:
    #    log.info("d4s_ctg_namespaces_controller instancing...")
    #    d4s_ctg_namespaces_controller = helpers.get_d4s_namespace_controller()
    #    log.info("d4s_ctg_namespaces_controller instancied %s" % d4s_ctg_namespaces_controller)


    #IFacets
    def dataset_facets(self, facets_dict, package_type):
        facets_dict = self._update_facets(facets_dict)
        return facets_dict
    
    def group_facets(self, facets_dict, group_type, package_type):
        facets_dict = self._update_facets(facets_dict)
        return facets_dict
    
    def organization_facets(self, facets_dict, organization_type, package_type):
        facets_dict = self._update_facets(facets_dict)
        return facets_dict
    
    def _update_facets(self, facets_dict):
        '''Add 'metadatatype' to facets if not already present.'''
        
        log.debug("facets_dict: ")
        log.debug(', '.join(facets_dict))
        
        metadatatype = helpers.get_systemtype_field_dict_from_session()

        '''Adding system:type'''
        facet_title = helpers.purge_namespace_to_fieldname(str(metadatatype['id']))
        facet_title = plugins.toolkit._(facet_title.capitalize() + 's')
        facets_dict = self._add_or_update_facet(metadatatype['name'],facet_title, facets_dict)
        
        log.info("site_url is: "+g.site_url)
        
        #ADD IT IN THE CUSTOMIZATION?
        if g.site_url:
            
            dev_sites = ['https://ckan-d-d4s.d4science.org']
            #GRSF Catalogues. 'Status of the Record' must be distributed everywhere, see #23398
            grsf_sites = ['https://ckan-grsf-admin2.d4science.org', 'https://ckan-grsf.pre.d4science.org', 'https://ckan-grsf.d4science.org', 'https://ckan-grsf-pre.d4science.org']
            sbd_sites = ['https://ckan.sobigdata.d4science.net', 'https://ckan-sobigdata.d4science.org',
                         'https://ckan-sobigdata2.d4science.org']

            if g.site_url in dev_sites:
                '''Adding Status of the GRSF record'''
                facets_dict = self._add_or_update_facet("StatusoftheRecord", "Status of the Record", facets_dict,
                                                        display_after_facet='groups')
                
                facets_dict = self._add_or_update_facet("Anno", "Anno", facets_dict,
                                                        display_after_facet='groups')
                    
            elif g.site_url in grsf_sites:
                '''Adding Status of the GRSF record'''
                # facets_dict = self._add_or_update_facet("StatusoftheGRSFrecord", "Status of the GRSF record", facets_dict, display_after_facet='groups')
                # Fixing #23348
                facets_dict = self._add_or_update_facet("StatusoftheRecord", "Status of the Record", facets_dict,
                                                        display_after_facet='groups')
            elif g.site_url in sbd_sites:
                '''Adding the field Availability '''
                facets_dict = self._add_or_update_facet("Availability", "Availability", facets_dict,
                                                        display_after_facet='groups')
        
        return facets_dict
    
    def before_map(self, map):
        """This IRoutes implementation overrides the standard
        ``/user/register`` behaviour with a custom controller.  You
        might instead use it to provide a completely new page, for
        example.
        Note that we have also provided a custom register form
        template at ``theme/templates/user/register.html``.
        """
        # Hook in our custom user controller at the points of creation
        # and edition.
        #
        #map.connect('/type', controller='ckanext.d4science_theme.controllers.type::d4STypeController', action='index')
        map.connect('/type', controller='ckanext.d4science_theme.controllers.systemtype:d4STypeController', action='index')
        ''' Added by Francesco Mangiacrapa, see: #8964 '''
        organization_vre = OrganizationVREController()
        map.connect('/organization_vre', controller='ckanext.d4science_theme.controllers.organization:OrganizationVREController', action='index')
        map.connect('/organization_vre/{id}', controller='ckanext.d4science_theme.controllers.organization:OrganizationVREController', action='read')
        map.redirect('/types', "/type")
        return map
        
    
    def _add_or_update_facet(self, facet_key, facet_value, facets_dict, display_after_facet='organization'):
        
        #Updating ordering of facets_dict OrderedDict
        if str(facet_key) not in facets_dict:

            new_orderded_facets_dict=facets_dict.__class__()
            for key, value in facets_dict.items():
                new_orderded_facets_dict[key]=value
                # #the field 'metadatatype' will be inserted after following key
                if key==display_after_facet:
                    new_orderded_facets_dict[facet_key]=facet_value
                    
            facets_dict.clear()
            facets_dict.update(new_orderded_facets_dict)
            log.debug("facets_dict ordered: ")
            log.debug(', '.join(facets_dict))
        
        return facets_dict