425 lines
19 KiB
Python
425 lines
19 KiB
Python
# encoding: utf-8
|
|
from logging import getLogger
|
|
|
|
import ckan.plugins as plugins
|
|
from ckanext.d4science_theme import helpers
|
|
import ckan.plugins.toolkit as toolkit
|
|
import ckan.lib.dictization.model_save as model_save
|
|
import ckan.model as model
|
|
import ckan.lib.helpers as h
|
|
import sqlalchemy as sa
|
|
from ckan.controllers.home import HomeController
|
|
from ckanext.d4science_theme.controllers.organization import OrganizationVREController
|
|
from ckanext.d4science_theme.controllers.home import d4SHomeController
|
|
from ckan.config.middleware.common_middleware import TrackingMiddleware
|
|
from ckan.common import session
|
|
from ckan.plugins import IRoutes
|
|
|
|
from ckan.common import (
|
|
g
|
|
)
|
|
|
|
# Created by Francesco Mangiacrapa
|
|
# francesco.mangiacrapa@isti.cnr.it
|
|
# ISTI-CNR Pisa (ITALY)
|
|
|
|
log = getLogger(__name__)
|
|
|
|
d4s_ctg_namespaces_controller = None
|
|
|
|
def remove_check_replicated_custom_key(schema):
|
|
if schema is not None:
|
|
schema.pop('__before', None)
|
|
|
|
return schema
|
|
|
|
#CREATED BY FRANCESCO MANGIACRAPA FOR OVERRIDING THE package_extras_save FROM dictization.model_save.py
|
|
def _package_extras_save(extra_dicts, obj, context):
|
|
''' It can save repeated extras as key-value '''
|
|
allow_partial_update = context.get("allow_partial_update", False)
|
|
if extra_dicts is None and allow_partial_update:
|
|
return
|
|
|
|
model = context["model"]
|
|
session = context["session"]
|
|
|
|
#ADDED BY FRANCESCO MANGIACRAPA
|
|
log.debug("extra_dicts: "+unicode(str(extra_dicts)).encode('utf-8'))
|
|
#print "extra_dicts: "+str(extra_dicts)
|
|
|
|
extras_list = obj.extras_list
|
|
#extras = dict((extra.key, extra) for extra in extras_list)
|
|
old_extras = {}
|
|
extras = {}
|
|
for extra in extras_list or []:
|
|
old_extras.setdefault(extra.key, []).append(extra.value)
|
|
extras.setdefault(extra.key, []).append(extra)
|
|
|
|
#ADDED BY FRANCESCO MANGIACRAPA
|
|
#print "old_extras: "+str(old_extras)
|
|
|
|
new_extras = {}
|
|
for extra_dict in extra_dicts or []:
|
|
#print 'extra_dict key: '+extra_dict["key"] + ', value: '+extra_dict["value"]
|
|
#new_extras.setdefault(extra_dict["key"], []).append(extra_dict["value"])
|
|
if extra_dict.get("deleted"):
|
|
log.debug("extra_dict deleted: "+unicode(extra_dict["key"]).encode('utf-8'))
|
|
#print 'extra_dict deleted: '+extra_dict["key"]
|
|
continue
|
|
|
|
#if extra_dict['value'] is not None and not extra_dict["value"] == "":
|
|
if extra_dict['value'] is not None:
|
|
new_extras.setdefault(extra_dict["key"], []).append(extra_dict["value"])
|
|
|
|
#ADDED BY FRANCESCO MANGIACRAPA
|
|
log.debug("new_extras: "+unicode(str(new_extras)).encode('utf-8'))
|
|
#print "new_extras: "+str(new_extras)
|
|
|
|
#new
|
|
for key in set(new_extras.keys()) - set(old_extras.keys()):
|
|
state = 'active'
|
|
log.debug("adding key: "+unicode(key).encode('utf-8'))
|
|
#print "adding key: "+str(key)
|
|
extra_lst = new_extras[key]
|
|
for extra in extra_lst:
|
|
extra = model.PackageExtra(state=state, key=key, value=extra)
|
|
session.add(extra)
|
|
extras_list.append(extra)
|
|
|
|
#deleted
|
|
for key in set(old_extras.keys()) - set(new_extras.keys()):
|
|
log.debug("deleting key: "+unicode(key).encode('utf-8'))
|
|
#print "deleting key: "+str(key)
|
|
extra_lst = extras[key]
|
|
for extra in extra_lst:
|
|
state = 'deleted'
|
|
extra.state = state
|
|
extras_list.remove(extra)
|
|
|
|
#changed
|
|
for key in set(new_extras.keys()) & set(old_extras.keys()):
|
|
#for each value of new list
|
|
for value in new_extras[key]:
|
|
old_occur = old_extras[key].count(value)
|
|
new_occur = new_extras[key].count(value)
|
|
log.debug("value: "+unicode(value).encode('utf-8') + ", new_occur: "+unicode(new_occur).encode('utf-8')+ ", old_occur: "+unicode(old_occur).encode('utf-8'))
|
|
#print "value: "+str(value) + ", new_occur: "+str(new_occur) + ", old_occur: "+str(old_occur)
|
|
# it is an old value deleted or not
|
|
if value in old_extras[key]:
|
|
if old_occur == new_occur:
|
|
#print "extra - occurrences of: "+str(value) +", are equal into both list"
|
|
log.debug("extra - occurrences of: "+unicode(value).encode('utf-8') +", are equal into both list")
|
|
#there is a little bug, this code return always the first element, so I'm fixing with #FIX-STATUS
|
|
extra_values = get_package_for_value(extras[key], value)
|
|
#extras_list.append(extra)
|
|
for extra in extra_values:
|
|
state = 'active'
|
|
extra.state = state
|
|
session.add(extra)
|
|
#print "extra updated: "+str(extra)
|
|
log.debug("extra updated: "+unicode(extra).encode('utf-8'))
|
|
|
|
elif new_occur > old_occur:
|
|
#print "extra - a new occurrence of: "+str(value) +", is present into new list, adding it to old list"
|
|
log.debug("extra - a new occurrence of: "+unicode(value).encode('utf-8') +", is present into new list, adding it to old list")
|
|
state = 'active'
|
|
extra = model.PackageExtra(state=state, key=key, value=value)
|
|
extra.state = state
|
|
session.add(extra)
|
|
extras_list.append(extra)
|
|
old_extras[key].append(value)
|
|
log.debug("old extra values updated: "+unicode(old_extras[key]).encode('utf-8'))
|
|
#print "old extra values updated: "+str(old_extras[key])
|
|
|
|
else:
|
|
#remove all occurrences deleted - this code could be optimized, it is run several times but could be performed one shot
|
|
countDelete = old_occur-new_occur
|
|
log.debug("extra - occurrence of: "+unicode(value).encode('utf-8')+", is not present into new list, removing "+unicode(countDelete).encode('utf-8')+" occurrence/s from old list")
|
|
#print "extra - occurrence of: "+str(value) +", is not present into new list, removing "+str(countDelete)+" occurrence/s from old list"
|
|
extra_values = get_package_for_value(extras[key], value)
|
|
for idx, extra in enumerate(extra_values):
|
|
if idx < countDelete:
|
|
#print "extra - occurrence of: "+str(value) +", is not present into new list, removing it from old list"
|
|
log.debug("pkg extra deleting: "+unicode(extra.value).encode('utf-8'))
|
|
#print "pkg extra deleting: "+str(extra.value)
|
|
state = 'deleted'
|
|
extra.state = state
|
|
|
|
else:
|
|
#print "pkg extra reactivating: "+str(extra.value)
|
|
log.debug("pkg extra reactivating: "+unicode(extra.value).encode('utf-8'))
|
|
state = 'active'
|
|
extra.state = state
|
|
session.add(extra)
|
|
|
|
else:
|
|
#print "extra new value: "+str(value)
|
|
log.debug("extra new value: "+unicode(value).encode('utf-8'))
|
|
state = 'active'
|
|
extra = model.PackageExtra(state=state, key=key, value=value)
|
|
extra.state = state
|
|
session.add(extra)
|
|
extras_list.append(extra)
|
|
|
|
|
|
#for each value of old list
|
|
for value in old_extras[key]:
|
|
#if value is not present in new list
|
|
if value not in new_extras[key]:
|
|
extra_values = get_package_for_value(extras[key], value)
|
|
for extra in extra_values:
|
|
#print "not present extra deleting: "+str(extra)
|
|
log.debug("not present extra deleting: "+unicode(extra).encode('utf-8'))
|
|
state = 'deleted'
|
|
extra.state = state
|
|
|
|
|
|
#ADDED BY FRANCESCO MANGIACRAPA
|
|
def get_package_for_value(list_package, value):
|
|
''' Returns a list of packages containing the value passed in input
|
|
'''
|
|
lst = []
|
|
for x in list_package:
|
|
if x.value == value:
|
|
lst.append(x)
|
|
else:
|
|
return lst
|
|
|
|
return lst
|
|
|
|
#OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE
|
|
def _init_TrackingMiddleware(self, app, config):
|
|
self.app = app
|
|
log.debug('TrackingMiddleware d4Science instance')
|
|
sqlalchemy_url = config.get('sqlalchemy.url')
|
|
log.debug('sqlalchemy_url read: '+str(sqlalchemy_url))
|
|
|
|
sqlalchemy_pool = config.get('sqlalchemy.pool_size')
|
|
if sqlalchemy_pool is None:
|
|
sqlalchemy_pool = 5
|
|
|
|
log.debug('sqlalchemy_pool read: '+str(sqlalchemy_pool))
|
|
sqlalchemy_overflow = config.get('sqlalchemy.max_overflow')
|
|
|
|
if sqlalchemy_overflow is None:
|
|
sqlalchemy_overflow = 10;
|
|
|
|
log.debug('sqlalchemy_overflow read: '+str(sqlalchemy_overflow))
|
|
|
|
try:
|
|
self.engine = sa.create_engine(sqlalchemy_url, pool_size=int(sqlalchemy_pool), max_overflow=int(sqlalchemy_overflow))
|
|
except TypeError as e:
|
|
log.error('pool size does not work: ' +str(e.args))
|
|
self.engine = sa.create_engine(sqlalchemy_url)
|
|
|
|
|
|
|
|
class D4Science_ThemePlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm):
|
|
plugins.implements(plugins.IConfigurer)
|
|
plugins.implements(plugins.IDatasetForm)
|
|
plugins.implements(plugins.ITemplateHelpers)
|
|
plugins.implements(plugins.IFacets)
|
|
plugins.implements(IRoutes, inherit=True)
|
|
|
|
# IConfigurer
|
|
def update_config(self, config_):
|
|
# Add this plugin's templates dir to CKAN's extra_template_paths, so
|
|
# that CKAN will use this plugin's custom templates.
|
|
toolkit.add_template_directory(config_, 'templates')
|
|
|
|
# Add this plugin's public dir to CKAN's extra_public_paths, so
|
|
# that CKAN will use this plugin's custom static files.
|
|
toolkit.add_public_directory(config_, 'public')
|
|
|
|
# Register this plugin's fanstatic directory with CKAN.
|
|
# Here, 'fanstatic' is the path to the fanstatic directory
|
|
# (relative to this plugin.py file), and 'example_theme' is the name
|
|
# that we'll use to refer to this fanstatic directory from CKAN
|
|
# templates.
|
|
toolkit.add_resource('fanstatic', 'd4science_theme')
|
|
|
|
#IDatasetForm
|
|
def create_package_schema(self):
|
|
# let's grab the default schema in our plugin
|
|
schema = super(D4Science_ThemePlugin, self).create_package_schema()
|
|
schema = remove_check_replicated_custom_key(schema)
|
|
#d.package_dict_save = _package_dict_save
|
|
return schema
|
|
|
|
#IDatasetForm
|
|
def update_package_schema(self):
|
|
schema = super(D4Science_ThemePlugin, self).update_package_schema()
|
|
schema = remove_check_replicated_custom_key(schema)
|
|
return schema
|
|
|
|
#IDatasetForm
|
|
def show_package_schema(self):
|
|
schema = super(D4Science_ThemePlugin, self).show_package_schema()
|
|
schema = remove_check_replicated_custom_key(schema)
|
|
return schema
|
|
|
|
#IDatasetForm
|
|
def is_fallback(self):
|
|
# Return True to register this plugin as the default handler for package types not handled by any other IDatasetForm plugin
|
|
return False
|
|
|
|
#IDatasetForm
|
|
def package_types(self):
|
|
# This plugin doesn't handle any special package types, it just
|
|
# registers itself as the default (above).
|
|
return []
|
|
|
|
|
|
#ITemplateHelpers
|
|
def get_helpers(self):
|
|
log.info("get_helpers called...")
|
|
'''Register functions as a template
|
|
helper function.
|
|
'''
|
|
# Template helper function names should begin with the name of the
|
|
# extension they belong to, to avoid clashing with functions from
|
|
# other extensions.
|
|
return {
|
|
'd4science_theme_get_user_role_for_group_or_org': helpers.get_user_role_for_group_or_org,
|
|
'd4science_theme_get_parents_for_group': helpers.get_parents_for_group,
|
|
'get_header_param': helpers.get_header_param,
|
|
'get_request_param': helpers.get_request_param,
|
|
'get_cookie_value': helpers.get_cookie_value,
|
|
'd4science_theme_markdown_extract_html' : helpers.markdown_extract_html,
|
|
'd4science_theme_get_systemtype_value_from_extras' : helpers.get_systemtype_value_from_extras,
|
|
'd4science_theme_get_systemtype_field_dict_from_session' : helpers.get_systemtype_field_dict_from_session,
|
|
'd4science_theme_get_namespace_separator_from_session' : helpers.get_namespace_separator_from_session,
|
|
'd4science_theme_get_extras' : helpers.get_extras,
|
|
'd4science_theme_count_facet_items_dict' : helpers.count_facet_items_dict,
|
|
'd4science_theme_purge_namespace_to_facet': helpers.purge_namespace_to_fieldname,
|
|
'd4science_get_color_for_type': helpers.get_color_for_type,
|
|
'd4science_get_d4s_namespace_controller': helpers.get_d4s_namespace_controller,
|
|
'd4science_get_extras_indexed_for_namespaces': helpers.get_extras_indexed_for_namespaces,
|
|
'd4science_get_namespaces_dict': helpers.get_namespaces_dict,
|
|
'd4science_get_extra_for_category' : helpers.get_extra_for_category,
|
|
'd4science_get_ordered_dictionary': helpers.ordered_dictionary,
|
|
'd4science_get_qrcode_for_url': helpers.qrcode_for_url,
|
|
'd4science_get_list_of_organizations': helpers.get_list_of_organizations,
|
|
'd4science_get_image_display_for_group': helpers.get_image_display_for_group,
|
|
'd4science_get_list_of_groups': helpers.get_list_of_groups,
|
|
'd4science_get_browse_info_for_organisations_or_groups': helpers.get_browse_info_for_organisations_or_groups,
|
|
'd4science_get_user_info': helpers.get_user_info,
|
|
'd4science_get_url_to_icon_for_ckan_entity' : helpers.get_url_to_icon_for_ckan_entity,
|
|
'd4science_get_ckan_translate_for' : helpers.get_ckan_translate_for,
|
|
'd4science_get_location_to_bboxes' : helpers.get_location_to_bboxes,
|
|
'd4science_get_content_moderator_system_placeholder': helpers.get_content_moderator_system_placeholder,
|
|
}
|
|
|
|
#Overriding package_extras_save method
|
|
model_save.package_extras_save = _package_extras_save
|
|
|
|
#Overriding index home controller
|
|
d4sHC = d4SHomeController()
|
|
HomeController.index = d4sHC.index
|
|
|
|
#OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE
|
|
TrackingMiddleware.__init__ = _init_TrackingMiddleware
|
|
|
|
global d4s_ctg_namespaces_controller
|
|
|
|
#if d4s_ctg_namespaces_controller is None:
|
|
# log.info("d4s_ctg_namespaces_controller instancing...")
|
|
# d4s_ctg_namespaces_controller = helpers.get_d4s_namespace_controller()
|
|
# log.info("d4s_ctg_namespaces_controller instancied %s" % d4s_ctg_namespaces_controller)
|
|
|
|
|
|
#IFacets
|
|
def dataset_facets(self, facets_dict, package_type):
|
|
facets_dict = self._update_facets(facets_dict)
|
|
return facets_dict
|
|
|
|
def group_facets(self, facets_dict, group_type, package_type):
|
|
facets_dict = self._update_facets(facets_dict)
|
|
return facets_dict
|
|
|
|
def organization_facets(self, facets_dict, organization_type, package_type):
|
|
facets_dict = self._update_facets(facets_dict)
|
|
return facets_dict
|
|
|
|
def _update_facets(self, facets_dict):
|
|
'''Add 'metadatatype' to facets if not already present.'''
|
|
|
|
log.debug("facets_dict: ")
|
|
log.debug(', '.join(facets_dict))
|
|
|
|
metadatatype = helpers.get_systemtype_field_dict_from_session()
|
|
|
|
'''Adding system:type'''
|
|
facet_title = helpers.purge_namespace_to_fieldname(str(metadatatype['id']))
|
|
facet_title = plugins.toolkit._(facet_title.capitalize() + 's')
|
|
facets_dict = self._add_or_update_facet(metadatatype['name'],facet_title, facets_dict)
|
|
|
|
log.info("site_url is: "+g.site_url)
|
|
|
|
#ADD IT IN THE CUSTOMIZATION?
|
|
if g.site_url:
|
|
|
|
dev_sites = ['https://ckan-d-d4s.d4science.org']
|
|
grsf_sites = ['https://ckan-grsf-admin2.d4science.org', 'https://ckan-grsf.pre.d4science.org']
|
|
sbd_sites = ['https://ckan.sobigdata.d4science.net', 'https://ckan-sobigdata.d4science.org',
|
|
'https://ckan-sobigdata2.d4science.org']
|
|
|
|
if g.site_url in dev_sites:
|
|
'''Adding Status of the GRSF record'''
|
|
facets_dict = self._add_or_update_facet("StatusoftheRecord", "Status of the Record", facets_dict,
|
|
display_after_facet='groups')
|
|
elif g.site_url in grsf_sites:
|
|
'''Adding Status of the GRSF record'''
|
|
# facets_dict = self._add_or_update_facet("StatusoftheGRSFrecord", "Status of the GRSF record", facets_dict, display_after_facet='groups')
|
|
# Fixing #23348
|
|
facets_dict = self._add_or_update_facet("StatusoftheRecord", "Status of the Record", facets_dict,
|
|
display_after_facet='groups')
|
|
elif g.site_url in sbd_sites:
|
|
'''Adding the field Availability '''
|
|
facets_dict = self._add_or_update_facet("Availability", "Availability", facets_dict,
|
|
display_after_facet='groups')
|
|
|
|
return facets_dict
|
|
|
|
def before_map(self, map):
|
|
"""This IRoutes implementation overrides the standard
|
|
``/user/register`` behaviour with a custom controller. You
|
|
might instead use it to provide a completely new page, for
|
|
example.
|
|
Note that we have also provided a custom register form
|
|
template at ``theme/templates/user/register.html``.
|
|
"""
|
|
# Hook in our custom user controller at the points of creation
|
|
# and edition.
|
|
#
|
|
#map.connect('/type', controller='ckanext.d4science_theme.controllers.type::d4STypeController', action='index')
|
|
map.connect('/type', controller='ckanext.d4science_theme.controllers.systemtype:d4STypeController', action='index')
|
|
''' Added by Francesco Mangiacrapa, see: #8964 '''
|
|
organization_vre = OrganizationVREController()
|
|
map.connect('/organization_vre', controller='ckanext.d4science_theme.controllers.organization:OrganizationVREController', action='index')
|
|
map.connect('/organization_vre/{id}', controller='ckanext.d4science_theme.controllers.organization:OrganizationVREController', action='read')
|
|
map.redirect('/types', "/type")
|
|
return map
|
|
|
|
|
|
def _add_or_update_facet(self, facet_key, facet_value, facets_dict, display_after_facet='organization'):
|
|
|
|
#Updating ordering of facets_dict OrderedDict
|
|
if str(facet_key) not in facets_dict:
|
|
|
|
new_orderded_facets_dict=facets_dict.__class__()
|
|
for key, value in facets_dict.items():
|
|
new_orderded_facets_dict[key]=value
|
|
# #the field 'metadatatype' will be inserted after following key
|
|
if key==display_after_facet:
|
|
new_orderded_facets_dict[facet_key]=facet_value
|
|
|
|
facets_dict.clear()
|
|
facets_dict.update(new_orderded_facets_dict)
|
|
log.debug("facets_dict ordered: ")
|
|
log.debug(', '.join(facets_dict))
|
|
|
|
return facets_dict
|
|
|
|
|