# encoding: utf-8 from logging import getLogger import ckan.plugins as plugins from ckanext.d4science_theme import helpers import ckan.plugins.toolkit as toolkit import ckan.lib.dictization.model_save as model_save import ckan.model as model import ckan.lib.helpers as h import sqlalchemy as sa from ckan.controllers.home import HomeController from ckanext.d4science_theme.controllers.organization import OrganizationVREController from ckanext.d4science_theme.controllers.home import d4SHomeController from ckan.config.middleware.common_middleware import TrackingMiddleware from ckan.common import session from ckan.plugins import IRoutes from ckan.common import ( g ) # Created by Francesco Mangiacrapa # francesco.mangiacrapa@isti.cnr.it # ISTI-CNR Pisa (ITALY) log = getLogger(__name__) d4s_ctg_namespaces_controller = None def remove_check_replicated_custom_key(schema): if schema is not None: schema.pop('__before', None) return schema #CREATED BY FRANCESCO MANGIACRAPA FOR OVERRIDING THE package_extras_save FROM dictization.model_save.py def _package_extras_save(extra_dicts, obj, context): ''' It can save repeated extras as key-value ''' allow_partial_update = context.get("allow_partial_update", False) if extra_dicts is None and allow_partial_update: return model = context["model"] session = context["session"] #ADDED BY FRANCESCO MANGIACRAPA log.debug("extra_dicts: "+unicode(str(extra_dicts)).encode('utf-8')) #print "extra_dicts: "+str(extra_dicts) extras_list = obj.extras_list #extras = dict((extra.key, extra) for extra in extras_list) old_extras = {} extras = {} for extra in extras_list or []: old_extras.setdefault(extra.key, []).append(extra.value) extras.setdefault(extra.key, []).append(extra) #ADDED BY FRANCESCO MANGIACRAPA #print "old_extras: "+str(old_extras) new_extras = {} for extra_dict in extra_dicts or []: #print 'extra_dict key: '+extra_dict["key"] + ', value: '+extra_dict["value"] #new_extras.setdefault(extra_dict["key"], []).append(extra_dict["value"]) if extra_dict.get("deleted"): log.debug("extra_dict deleted: "+unicode(extra_dict["key"]).encode('utf-8')) #print 'extra_dict deleted: '+extra_dict["key"] continue #if extra_dict['value'] is not None and not extra_dict["value"] == "": if extra_dict['value'] is not None: new_extras.setdefault(extra_dict["key"], []).append(extra_dict["value"]) #ADDED BY FRANCESCO MANGIACRAPA log.debug("new_extras: "+unicode(str(new_extras)).encode('utf-8')) #print "new_extras: "+str(new_extras) #new for key in set(new_extras.keys()) - set(old_extras.keys()): state = 'active' log.debug("adding key: "+unicode(key).encode('utf-8')) #print "adding key: "+str(key) extra_lst = new_extras[key] for extra in extra_lst: extra = model.PackageExtra(state=state, key=key, value=extra) session.add(extra) extras_list.append(extra) #deleted for key in set(old_extras.keys()) - set(new_extras.keys()): log.debug("deleting key: "+unicode(key).encode('utf-8')) #print "deleting key: "+str(key) extra_lst = extras[key] for extra in extra_lst: state = 'deleted' extra.state = state extras_list.remove(extra) #changed for key in set(new_extras.keys()) & set(old_extras.keys()): #for each value of new list for value in new_extras[key]: old_occur = old_extras[key].count(value) new_occur = new_extras[key].count(value) log.debug("value: "+unicode(value).encode('utf-8') + ", new_occur: "+unicode(new_occur).encode('utf-8')+ ", old_occur: "+unicode(old_occur).encode('utf-8')) #print "value: "+str(value) + ", new_occur: "+str(new_occur) + ", old_occur: "+str(old_occur) # it is an old value deleted or not if value in old_extras[key]: if old_occur == new_occur: #print "extra - occurrences of: "+str(value) +", are equal into both list" log.debug("extra - occurrences of: "+unicode(value).encode('utf-8') +", are equal into both list") #there is a little bug, this code return always the first element, so I'm fixing with #FIX-STATUS extra_values = get_package_for_value(extras[key], value) #extras_list.append(extra) for extra in extra_values: state = 'active' extra.state = state session.add(extra) #print "extra updated: "+str(extra) log.debug("extra updated: "+unicode(extra).encode('utf-8')) elif new_occur > old_occur: #print "extra - a new occurrence of: "+str(value) +", is present into new list, adding it to old list" log.debug("extra - a new occurrence of: "+unicode(value).encode('utf-8') +", is present into new list, adding it to old list") state = 'active' extra = model.PackageExtra(state=state, key=key, value=value) extra.state = state session.add(extra) extras_list.append(extra) old_extras[key].append(value) log.debug("old extra values updated: "+unicode(old_extras[key]).encode('utf-8')) #print "old extra values updated: "+str(old_extras[key]) else: #remove all occurrences deleted - this code could be optimized, it is run several times but could be performed one shot countDelete = old_occur-new_occur log.debug("extra - occurrence of: "+unicode(value).encode('utf-8')+", is not present into new list, removing "+unicode(countDelete).encode('utf-8')+" occurrence/s from old list") #print "extra - occurrence of: "+str(value) +", is not present into new list, removing "+str(countDelete)+" occurrence/s from old list" extra_values = get_package_for_value(extras[key], value) for idx, extra in enumerate(extra_values): if idx < countDelete: #print "extra - occurrence of: "+str(value) +", is not present into new list, removing it from old list" log.debug("pkg extra deleting: "+unicode(extra.value).encode('utf-8')) #print "pkg extra deleting: "+str(extra.value) state = 'deleted' extra.state = state else: #print "pkg extra reactivating: "+str(extra.value) log.debug("pkg extra reactivating: "+unicode(extra.value).encode('utf-8')) state = 'active' extra.state = state session.add(extra) else: #print "extra new value: "+str(value) log.debug("extra new value: "+unicode(value).encode('utf-8')) state = 'active' extra = model.PackageExtra(state=state, key=key, value=value) extra.state = state session.add(extra) extras_list.append(extra) #for each value of old list for value in old_extras[key]: #if value is not present in new list if value not in new_extras[key]: extra_values = get_package_for_value(extras[key], value) for extra in extra_values: #print "not present extra deleting: "+str(extra) log.debug("not present extra deleting: "+unicode(extra).encode('utf-8')) state = 'deleted' extra.state = state #ADDED BY FRANCESCO MANGIACRAPA def get_package_for_value(list_package, value): ''' Returns a list of packages containing the value passed in input ''' lst = [] for x in list_package: if x.value == value: lst.append(x) else: return lst return lst #OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE def _init_TrackingMiddleware(self, app, config): self.app = app log.debug('TrackingMiddleware d4Science instance') sqlalchemy_url = config.get('sqlalchemy.url') log.debug('sqlalchemy_url read: '+str(sqlalchemy_url)) sqlalchemy_pool = config.get('sqlalchemy.pool_size') if sqlalchemy_pool is None: sqlalchemy_pool = 5 log.debug('sqlalchemy_pool read: '+str(sqlalchemy_pool)) sqlalchemy_overflow = config.get('sqlalchemy.max_overflow') if sqlalchemy_overflow is None: sqlalchemy_overflow = 10; log.debug('sqlalchemy_overflow read: '+str(sqlalchemy_overflow)) try: self.engine = sa.create_engine(sqlalchemy_url, pool_size=int(sqlalchemy_pool), max_overflow=int(sqlalchemy_overflow)) except TypeError as e: log.error('pool size does not work: ' +str(e.args)) self.engine = sa.create_engine(sqlalchemy_url) class D4Science_ThemePlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm): plugins.implements(plugins.IConfigurer) plugins.implements(plugins.IDatasetForm) plugins.implements(plugins.ITemplateHelpers) plugins.implements(plugins.IFacets) plugins.implements(IRoutes, inherit=True) # IConfigurer def update_config(self, config_): # Add this plugin's templates dir to CKAN's extra_template_paths, so # that CKAN will use this plugin's custom templates. toolkit.add_template_directory(config_, 'templates') # Add this plugin's public dir to CKAN's extra_public_paths, so # that CKAN will use this plugin's custom static files. toolkit.add_public_directory(config_, 'public') # Register this plugin's fanstatic directory with CKAN. # Here, 'fanstatic' is the path to the fanstatic directory # (relative to this plugin.py file), and 'example_theme' is the name # that we'll use to refer to this fanstatic directory from CKAN # templates. toolkit.add_resource('fanstatic', 'd4science_theme') #IDatasetForm def create_package_schema(self): # let's grab the default schema in our plugin schema = super(D4Science_ThemePlugin, self).create_package_schema() schema = remove_check_replicated_custom_key(schema) #d.package_dict_save = _package_dict_save return schema #IDatasetForm def update_package_schema(self): schema = super(D4Science_ThemePlugin, self).update_package_schema() schema = remove_check_replicated_custom_key(schema) return schema #IDatasetForm def show_package_schema(self): schema = super(D4Science_ThemePlugin, self).show_package_schema() schema = remove_check_replicated_custom_key(schema) return schema #IDatasetForm def is_fallback(self): # Return True to register this plugin as the default handler for package types not handled by any other IDatasetForm plugin return False #IDatasetForm def package_types(self): # This plugin doesn't handle any special package types, it just # registers itself as the default (above). return [] #ITemplateHelpers def get_helpers(self): log.info("get_helpers called...") '''Register functions as a template helper function. ''' # Template helper function names should begin with the name of the # extension they belong to, to avoid clashing with functions from # other extensions. return { 'd4science_theme_get_user_role_for_group_or_org': helpers.get_user_role_for_group_or_org, 'd4science_theme_get_parents_for_group': helpers.get_parents_for_group, 'get_header_param': helpers.get_header_param, 'get_request_param': helpers.get_request_param, 'get_cookie_value': helpers.get_cookie_value, 'd4science_theme_markdown_extract_html' : helpers.markdown_extract_html, 'd4science_theme_get_systemtype_value_from_extras' : helpers.get_systemtype_value_from_extras, 'd4science_theme_get_systemtype_field_dict_from_session' : helpers.get_systemtype_field_dict_from_session, 'd4science_theme_get_namespace_separator_from_session' : helpers.get_namespace_separator_from_session, 'd4science_theme_get_extras' : helpers.get_extras, 'd4science_theme_count_facet_items_dict' : helpers.count_facet_items_dict, 'd4science_theme_purge_namespace_to_facet': helpers.purge_namespace_to_fieldname, 'd4science_get_color_for_type': helpers.get_color_for_type, 'd4science_get_d4s_namespace_controller': helpers.get_d4s_namespace_controller, 'd4science_get_extras_indexed_for_namespaces': helpers.get_extras_indexed_for_namespaces, 'd4science_get_namespaces_dict': helpers.get_namespaces_dict, 'd4science_get_extra_for_category' : helpers.get_extra_for_category, 'd4science_get_ordered_dictionary': helpers.ordered_dictionary, 'd4science_get_qrcode_for_url': helpers.qrcode_for_url, 'd4science_get_list_of_organizations': helpers.get_list_of_organizations, 'd4science_get_image_display_for_group': helpers.get_image_display_for_group, 'd4science_get_list_of_groups': helpers.get_list_of_groups, 'd4science_get_browse_info_for_organisations_or_groups': helpers.get_browse_info_for_organisations_or_groups, 'd4science_get_user_info': helpers.get_user_info, 'd4science_get_url_to_icon_for_ckan_entity' : helpers.get_url_to_icon_for_ckan_entity, 'd4science_get_ckan_translate_for' : helpers.get_ckan_translate_for, 'd4science_get_location_to_bboxes' : helpers.get_location_to_bboxes, 'd4science_get_content_moderator_system_placeholder': helpers.get_content_moderator_system_placeholder, } #Overriding package_extras_save method model_save.package_extras_save = _package_extras_save #Overriding index home controller d4sHC = d4SHomeController() HomeController.index = d4sHC.index #OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE TrackingMiddleware.__init__ = _init_TrackingMiddleware global d4s_ctg_namespaces_controller #if d4s_ctg_namespaces_controller is None: # log.info("d4s_ctg_namespaces_controller instancing...") # d4s_ctg_namespaces_controller = helpers.get_d4s_namespace_controller() # log.info("d4s_ctg_namespaces_controller instancied %s" % d4s_ctg_namespaces_controller) #IFacets def dataset_facets(self, facets_dict, package_type): facets_dict = self._update_facets(facets_dict) return facets_dict def group_facets(self, facets_dict, group_type, package_type): facets_dict = self._update_facets(facets_dict) return facets_dict def organization_facets(self, facets_dict, organization_type, package_type): facets_dict = self._update_facets(facets_dict) return facets_dict def _update_facets(self, facets_dict): '''Add 'metadatatype' to facets if not already present.''' log.debug("facets_dict: ") log.debug(', '.join(facets_dict)) metadatatype = helpers.get_systemtype_field_dict_from_session() '''Adding system:type''' facet_title = helpers.purge_namespace_to_fieldname(str(metadatatype['id'])) facet_title = plugins.toolkit._(facet_title.capitalize() + 's') facets_dict = self._add_or_update_facet(metadatatype['name'],facet_title, facets_dict) log.info("site_url is: "+g.site_url) #ADD IT IN THE CUSTOMIZATION? if g.site_url: dev_sites = ['https://ckan-d-d4s.d4science.org'] #GRSF Catalogues. 'Status of the Record' must be distributed everywhere, see #23398 grsf_sites = ['https://ckan-grsf-admin2.d4science.org', 'https://ckan-grsf.pre.d4science.org', 'https://ckan-grsf.d4science.org', 'https://ckan-grsf-pre.d4science.org'] sbd_sites = ['https://ckan.sobigdata.d4science.net', 'https://ckan-sobigdata.d4science.org', 'https://ckan-sobigdata2.d4science.org'] if g.site_url in dev_sites: '''Adding Status of the GRSF record''' facets_dict = self._add_or_update_facet("StatusoftheRecord", "Status of the Record", facets_dict, display_after_facet='groups') facets_dict = self._add_or_update_facet("Anno", "Anno", facets_dict, display_after_facet='groups') elif g.site_url in grsf_sites: '''Adding Status of the GRSF record''' # facets_dict = self._add_or_update_facet("StatusoftheGRSFrecord", "Status of the GRSF record", facets_dict, display_after_facet='groups') # Fixing #23348 facets_dict = self._add_or_update_facet("StatusoftheRecord", "Status of the Record", facets_dict, display_after_facet='groups') elif g.site_url in sbd_sites: '''Adding the field Availability ''' facets_dict = self._add_or_update_facet("Availability", "Availability", facets_dict, display_after_facet='groups') return facets_dict def before_map(self, map): """This IRoutes implementation overrides the standard ``/user/register`` behaviour with a custom controller. You might instead use it to provide a completely new page, for example. Note that we have also provided a custom register form template at ``theme/templates/user/register.html``. """ # Hook in our custom user controller at the points of creation # and edition. # #map.connect('/type', controller='ckanext.d4science_theme.controllers.type::d4STypeController', action='index') map.connect('/type', controller='ckanext.d4science_theme.controllers.systemtype:d4STypeController', action='index') ''' Added by Francesco Mangiacrapa, see: #8964 ''' organization_vre = OrganizationVREController() map.connect('/organization_vre', controller='ckanext.d4science_theme.controllers.organization:OrganizationVREController', action='index') map.connect('/organization_vre/{id}', controller='ckanext.d4science_theme.controllers.organization:OrganizationVREController', action='read') map.redirect('/types', "/type") return map def _add_or_update_facet(self, facet_key, facet_value, facets_dict, display_after_facet='organization'): #Updating ordering of facets_dict OrderedDict if str(facet_key) not in facets_dict: new_orderded_facets_dict=facets_dict.__class__() for key, value in facets_dict.items(): new_orderded_facets_dict[key]=value # #the field 'metadatatype' will be inserted after following key if key==display_after_facet: new_orderded_facets_dict[facet_key]=facet_value facets_dict.clear() facets_dict.update(new_orderded_facets_dict) log.debug("facets_dict ordered: ") log.debug(', '.join(facets_dict)) return facets_dict