convert to python3
This commit is contained in:
parent
89b1d0a4a3
commit
00c0bedacd
|
|
@ -1,13 +1,762 @@
|
|||
|
||||
def d4science_hello():
|
||||
return "Hello, d4science!"
|
||||
import ckan.authz as authz
|
||||
import ckan.model as model
|
||||
from webhelpers.html import escape, HTML, literal, url_escape
|
||||
from webhelpers.text import truncate
|
||||
import ckan.lib.helpers as h
|
||||
import ckan.logic as logic
|
||||
from ckan.common import config
|
||||
from ckanext.d4science.d4sdiscovery.d4s_namespaces_controller import D4S_Namespaces_Controller
|
||||
from ckanext.d4science.d4sdiscovery.d4s_namespaces_extras_util import D4S_Namespaces_Extra_Util
|
||||
from ckanext.d4science.qrcodelink.generate_qrcode import D4S_QrCode
|
||||
import urllib.request, urllib.error, urllib.parse
|
||||
|
||||
from ckan.common import (
|
||||
_, ungettext, g, c, request, session, json, OrderedDict
|
||||
)
|
||||
|
||||
from flask import Blueprint, render_template, g, request, url_for, current_app
|
||||
|
||||
import random
|
||||
from operator import itemgetter
|
||||
from logging import getLogger
|
||||
import base64
|
||||
import sys, os, re
|
||||
import configparser
|
||||
import collections
|
||||
|
||||
log = getLogger(__name__)
|
||||
|
||||
systemtype_field = 'systemtypefield'
|
||||
systemtype_field_default_value = 'system:type'
|
||||
ic_proxy_url_field = 'ic_proxy_url'
|
||||
ic_proxy_url_field_default_value = "https://registry.d4science.org/icproxy/gcube/service"
|
||||
application_token_field = 'application_token'
|
||||
namespaces_generic_resource_id_default_value = "23d827cd-ba8e-4d8c-9ab4-6303bdb7d1db"
|
||||
namespaces_gr_id_fieldname = "namespaces_generic_resource_id"
|
||||
namespaceseparator_field = 'namespace_separator'
|
||||
namespaceseparator_field_default_value = ':'
|
||||
systemtype_rgb_colors = ['#c0392b ', '#585858', '#04407C', '#9b59b6', '#2ecc71', '#16a085', '#7f8c8d ', '#2ecc71',
|
||||
'#FA8072', '#00FFFF', '#C76611', '#f39c12', '#800000']
|
||||
systemtype_field_colors = 'systemtype_field_colors'
|
||||
|
||||
systemtype_cms_fields_placeholders = {'prefix': 'system:cm_', 'item_status': 'system:cm_item_status'}
|
||||
|
||||
NOCATEOGORY = 'nocategory'
|
||||
TRANSLATE_OF_ = 'translate_of_'
|
||||
|
||||
ctg_namespace_ctrl = None
|
||||
|
||||
|
||||
def get_helpers():
|
||||
def get_user_role_for_group_or_org(group_id, user_name):
|
||||
''' Returns the user's role for the group. (Ignores privileges that cascade
|
||||
in a group hierarchy.)'''
|
||||
return authz.users_role_for_group_or_org(group_id, user_name)
|
||||
|
||||
def get_parents_for_group(group_name_or_id):
|
||||
''' Returns the user's role for the group. (Ignores privileges that cascade
|
||||
in a group hierarchy.)'''
|
||||
group = model.Group.get(group_name_or_id)
|
||||
if group:
|
||||
return model.Group.get_parent_group_hierarchy(group)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def get_header_param(parameter_name, default=None):
|
||||
''' This function allows templates to access header string parameters
|
||||
from the request. '''
|
||||
return request.headers.get(parameter_name, default)
|
||||
|
||||
|
||||
def get_request_param(parameter_name, default=None):
|
||||
''' This function allows templates to access query string parameters
|
||||
from the request. '''
|
||||
return request.args.get(parameter_name, default)
|
||||
#return request.params.get(parameter_name, default)
|
||||
|
||||
|
||||
def get_cookie_value(cookie_name, default=None):
|
||||
''' This function allows templates to access cookie by cookie_name parameter
|
||||
from the request. '''
|
||||
|
||||
value = request.cookies.get(cookie_name)
|
||||
|
||||
if value is None:
|
||||
print('cookie: ' + cookie_name + ', has value None')
|
||||
else:
|
||||
print('cookie: ' + cookie_name + ', has value ' + value)
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def markdown_extract_html(text, extract_length=190, allow_html=False):
|
||||
''' Returns the plain text representation of markdown encoded text. That
|
||||
is the texted without any html tags. If extract_length is 0 then it
|
||||
will not be truncated.'''
|
||||
if not text:
|
||||
return ''
|
||||
if allow_html:
|
||||
plain = h.markdown(text.strip())
|
||||
else:
|
||||
plain = h.RE_MD_HTML_TAGS.sub('', h.markdown(text))
|
||||
|
||||
if not extract_length or len(plain) < extract_length:
|
||||
return literal(plain)
|
||||
return literal(str(truncate(plain, length=extract_length, indicator='...', whole_word=True)))
|
||||
|
||||
|
||||
def get_systemtype_field_dict_from_session():
|
||||
'''Return the value of 'ckan.d4science_theme.metadatatypefield'
|
||||
read from production.ini'''
|
||||
|
||||
systemtype_fieldname = session.get(systemtype_field)
|
||||
|
||||
if systemtype_fieldname is None:
|
||||
log.info(systemtype_field + " not found in session, loading from config")
|
||||
else:
|
||||
log.debug(systemtype_field + " found in session having value: %s" % systemtype_fieldname)
|
||||
return systemtype_fieldname
|
||||
|
||||
systemtype_fieldname = current_app.config.get('ckan.d4science_theme.' + systemtype_field)
|
||||
#systemtype_fieldname = config.get('ckan.d4science_theme.' + systemtype_field)
|
||||
|
||||
if systemtype_fieldname is None:
|
||||
log.info(
|
||||
systemtype_field + " field does not exist in production.ini, returning default value %s" % systemtype_field_default_value)
|
||||
systemtype_fieldname = systemtype_field_default_value
|
||||
|
||||
separator = get_namespace_separator_from_session()
|
||||
log.debug("Replacing %s" % separator + " with empty string for key %s" % systemtype_field)
|
||||
systemtype_fieldname_name = systemtype_fieldname.replace(separator, "")
|
||||
purgedfieldname = purge_namespace_to_fieldname(systemtype_fieldname)
|
||||
log.debug("Setting %s" % systemtype_fieldname + " in session for key %s" % systemtype_field)
|
||||
session[systemtype_field] = {'id': systemtype_fieldname, 'name': systemtype_fieldname_name,
|
||||
'title': purgedfieldname}
|
||||
session.modified = True
|
||||
#session.save() old pylons(?)
|
||||
return session[systemtype_field]
|
||||
|
||||
|
||||
def get_d4s_namespace_controller():
|
||||
'''Instance the D4S_Namespaces_Controller and check that the namespaces are not empty reading it from IS and/or using a Caching system.
|
||||
The ic-proxy-url is built by reading the configurations from production.ini'''
|
||||
|
||||
d4s_extras_controller = D4S_Namespaces_Controller.getInstance()
|
||||
global ctg_namespace_ctrl
|
||||
|
||||
if ctg_namespace_ctrl is not None:
|
||||
log.info("ctg_namespace_ctrl with configurations is NOT None")
|
||||
the_namespaces = d4s_extras_controller.load_namespaces(ctg_namespace_ctrl['ic_proxy_url'],
|
||||
ctg_namespace_ctrl['resource_id'],
|
||||
ctg_namespace_ctrl['application_token'])
|
||||
log.debug("the_namespaces are %s" % the_namespaces)
|
||||
|
||||
if the_namespaces is None or len(the_namespaces) == 0:
|
||||
log.info("D4S_Namespaces_Controller obj with none or empty namespaces, going to read them")
|
||||
else:
|
||||
log.info("d4s_namespaces_controller found and the namespaces property is not empty: %s" % d4s_extras_controller)
|
||||
return d4s_extras_controller
|
||||
else:
|
||||
log.info("ctg_namespace_ctrl with configurations is None, instancing it")
|
||||
|
||||
ic_proxy_url_value = current_app.config.get('ckan.d4science_theme.' + ic_proxy_url_field)
|
||||
#ic_proxy_url_value = config.get('ckan.d4science_theme.' + ic_proxy_url_field) old
|
||||
|
||||
if ic_proxy_url_value is None:
|
||||
log.info(
|
||||
"ckan.d4science_theme." + ic_proxy_url_field + " field does not exist in production.ini, returning default value %s" % ic_proxy_url_field_default_value)
|
||||
ic_proxy_url_value = ic_proxy_url_field_default_value
|
||||
|
||||
application_token_fieldname = current_app.config.get('ckan.d4science_theme.' + application_token_field)
|
||||
#application_token_fieldname = config.get('ckan.d4science_theme.' + application_token_field)
|
||||
|
||||
if application_token_fieldname is None:
|
||||
log.error("ckan.d4science_theme." + application_token_field + " field does not exist in production.ini!!!")
|
||||
application_token_fieldname = None
|
||||
|
||||
namespaces_gr_id_fieldname_value = current_app.config.get('ckan.d4science_theme.' + namespaces_gr_id_fieldname)
|
||||
#namespaces_gr_id_fieldname_value = config.get('ckan.d4science_theme.' + namespaces_gr_id_fieldname)
|
||||
|
||||
if namespaces_gr_id_fieldname_value is None:
|
||||
log.error("ckan.d4science_theme." + application_token_field + " field does not exist in production.ini!!!")
|
||||
namespaces_gr_id_fieldname_value = namespaces_generic_resource_id_default_value
|
||||
|
||||
# filling the ctg_namespace_ctrl with IS configurations to perform the query for loading the namespaces from IS
|
||||
ctg_namespace_ctrl = {'ic_proxy_url': ic_proxy_url_value,
|
||||
'application_token': application_token_fieldname,
|
||||
'resource_id': namespaces_gr_id_fieldname_value}
|
||||
|
||||
d4s_extras_controller.load_namespaces(ctg_namespace_ctrl['ic_proxy_url'], ctg_namespace_ctrl['resource_id'],
|
||||
ctg_namespace_ctrl['application_token'])
|
||||
|
||||
return d4s_extras_controller
|
||||
|
||||
|
||||
def get_extras_indexed_for_namespaces(extras):
|
||||
namespace_dict = get_namespaces_dict()
|
||||
# log.info("my_namespace_dict %s" % namespace_dict)
|
||||
my_extra = get_extras(extras)
|
||||
# log.info("my_extra is %s" % my_extra)
|
||||
# d4s_extras_controller = D4S_Namespaces_Controller.getInstance()
|
||||
# extras_indexed_for_categories = d4s_extras_controller.get_extras_indexed_for_namespaces(namespace_dict, my_extra)
|
||||
|
||||
extras_indexed_for_categories = D4S_Namespaces_Extra_Util().get_extras_indexed_for_namespaces(namespace_dict,
|
||||
my_extra)
|
||||
return extras_indexed_for_categories
|
||||
|
||||
|
||||
def get_namespaces_dict():
|
||||
d4s_extras_controller = get_d4s_namespace_controller()
|
||||
|
||||
if d4s_extras_controller is not None:
|
||||
return d4s_extras_controller.get_dict_ctg_namespaces()
|
||||
else:
|
||||
log.info("local_extras_controller is null, returning empty dictionary for namespaces")
|
||||
return {}
|
||||
|
||||
|
||||
def get_extra_for_category(extras_indexed_for_categories, key_category):
|
||||
if key_category in extras_indexed_for_categories:
|
||||
catalogue_namespace = extras_indexed_for_categories[key_category]
|
||||
return catalogue_namespace.extras
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def get_systemtype_value_from_extras(package, extras=None):
|
||||
'''Returns the value of metadata fied read from key 'metadatatype'
|
||||
stored into extra fields if it exists, 'No Type' otherwise'''
|
||||
systemtype_dict = get_systemtype_field_dict_from_session()
|
||||
|
||||
no_type = 'No Type'
|
||||
|
||||
if extras is None:
|
||||
return no_type
|
||||
|
||||
for extra in extras:
|
||||
k, v = extra['key'], extra['value']
|
||||
log.debug("key is %s" % k)
|
||||
log.debug("value is %s" % v)
|
||||
if k == str(systemtype_dict['id']):
|
||||
return v
|
||||
|
||||
return no_type
|
||||
|
||||
|
||||
def get_namespace_separator_from_session():
|
||||
'''Returns the character used to separate namespace from fieldname'''
|
||||
|
||||
separator = session.get(namespaceseparator_field)
|
||||
|
||||
if separator is None:
|
||||
log.info(namespaceseparator_field + " not found in session, loading from config")
|
||||
else:
|
||||
log.debug(namespaceseparator_field + " found in session: %s" % separator)
|
||||
return separator
|
||||
|
||||
namespace_sep = config.get('ckan.d4science_theme.' + namespaceseparator_field)
|
||||
|
||||
if namespace_sep is None:
|
||||
log.info(
|
||||
namespaceseparator_field + " field does not exist in production.ini, returning default value %s" % namespaceseparator_field_default_value)
|
||||
namespace_sep = namespaceseparator_field_default_value
|
||||
|
||||
log.debug("Setting %s" % namespace_sep + " in session for key %s" % namespaceseparator_field)
|
||||
session[namespaceseparator_field] = namespace_sep
|
||||
return namespace_sep
|
||||
|
||||
|
||||
def get_extras(package_extras, auto_clean=False, subs=None, exclude=None):
|
||||
''' Used for outputting package extras
|
||||
|
||||
:param package_extras: the package extras
|
||||
:type package_extras: dict
|
||||
:param auto_clean: If true capitalize and replace -_ with spaces
|
||||
:type auto_clean: bool
|
||||
:param subs: substitutes to use instead of given keys
|
||||
:type subs: dict {'key': 'replacement'}
|
||||
:param exclude: keys to exclude
|
||||
:type exclude: list of strings
|
||||
'''
|
||||
|
||||
# If exclude is not supplied use values defined in the config
|
||||
if not exclude:
|
||||
exclude = g.package_hide_extras
|
||||
output = []
|
||||
for extra in package_extras:
|
||||
if extra.get('state') == 'deleted':
|
||||
continue
|
||||
k, v = extra['key'], extra['value']
|
||||
if k in exclude:
|
||||
continue
|
||||
if subs and k in subs:
|
||||
k = subs[k]
|
||||
elif auto_clean:
|
||||
k = k.replace('_', ' ').replace('-', ' ').title()
|
||||
if isinstance(v, (list, tuple)):
|
||||
v = ", ".join(map(str, v))
|
||||
output.append((k, v))
|
||||
return output
|
||||
|
||||
|
||||
def purge_namespace_to_fieldname(fieldname):
|
||||
separator = get_namespace_separator_from_session()
|
||||
|
||||
if fieldname is None:
|
||||
return ""
|
||||
|
||||
if separator not in fieldname:
|
||||
return fieldname
|
||||
|
||||
end = fieldname.index(separator) + 1
|
||||
max_l = len(fieldname)
|
||||
if end < max_l:
|
||||
return fieldname[end:max_l]
|
||||
return fieldname
|
||||
|
||||
|
||||
def purge_namespace_to_string(facet):
|
||||
if not g.search_facets or \
|
||||
not g.search_facets.get(facet) or \
|
||||
not g.search_facets.get(facet).get('items'):
|
||||
return ""
|
||||
|
||||
facet_name = g.search_facets.get(facet)
|
||||
print("facet_name " + str(facet_name))
|
||||
#logging.debug("facet_name " + str(facet_name)) preferibile in flask
|
||||
end = str(facet_name).index(":")
|
||||
if end <= len(facet_name):
|
||||
return facet_name[:end]
|
||||
return facet_name
|
||||
|
||||
|
||||
def count_facet_items_dict(facet, limit=None, exclude_active=False):
|
||||
if not g.search_facets or \
|
||||
not g.search_facets.get(facet) or \
|
||||
not g.search_facets.get(facet).get('items'):
|
||||
return 0
|
||||
facets = []
|
||||
for facet_item in g.search_facets.get(facet)['items']:
|
||||
if not len(facet_item['name'].strip()):
|
||||
continue
|
||||
if not (facet, facet_item['name']) in list(request.args.items()):
|
||||
facets.append(dict(active=False, **facet_item))
|
||||
elif not exclude_active:
|
||||
facets.append(dict(active=True, **facet_item))
|
||||
|
||||
# for count,
|
||||
# print "facets " + str(facets)
|
||||
total = len(facets)
|
||||
log.debug("total facet: %s are %d" % (facet, total))
|
||||
return total
|
||||
|
||||
|
||||
def random_color():
|
||||
rgbl = [255, 0, 0]
|
||||
random.shuffle(rgbl)
|
||||
return tuple(rgbl)
|
||||
|
||||
|
||||
def check_url(the_url):
|
||||
try:
|
||||
urllib.request.urlopen(the_url)
|
||||
return True
|
||||
except urllib.error.HTTPError as e:
|
||||
# print(e.code)
|
||||
return False
|
||||
except urllib.error.URLError as e:
|
||||
# print(e.args)
|
||||
return False
|
||||
except Exception as error:
|
||||
# print(error)
|
||||
return False
|
||||
|
||||
|
||||
def get_color_for_type(systemtype_field_value):
|
||||
'''Return a color assigned to a system type'''
|
||||
|
||||
systemtypecolors = session.get(systemtype_field_colors)
|
||||
# log.info("color: getting color for type: %s" %systemtype_field_value)
|
||||
|
||||
if systemtypecolors is None:
|
||||
log.info("color: " + systemtype_field_colors + " not found in session, creating new one")
|
||||
systemtypecolors = {}
|
||||
session[systemtype_field_colors] = systemtypecolors
|
||||
else:
|
||||
log.debug("color: " + systemtype_field_colors + " found in session having value: %s" % systemtypecolors)
|
||||
|
||||
e_color = systemtypecolors.get(systemtype_field_value)
|
||||
|
||||
if e_color is None:
|
||||
usedcolorsLen = len(systemtypecolors)
|
||||
colorsLen = len(systemtype_rgb_colors)
|
||||
index = usedcolorsLen if usedcolorsLen < colorsLen else random.randint(0, colorsLen - 1)
|
||||
e_color = systemtype_rgb_colors[index]
|
||||
# log.debug("color: adding color %s" %e_color +" index is: "+str(index))
|
||||
systemtypecolors[systemtype_field_value] = e_color
|
||||
session[systemtype_field_colors] = systemtypecolors
|
||||
|
||||
session.modified = True
|
||||
#session.save()
|
||||
# log.debug("color: returning color %s" %e_color +" for type: "+systemtype_field_value)
|
||||
return e_color
|
||||
|
||||
|
||||
def ordered_dictionary(list_to_be_sorted, property='name', ordering="asc"):
|
||||
# print ("dict %s" %list_to_be_sorted)
|
||||
|
||||
ord = False if ordering == "asc" else True
|
||||
|
||||
if list_to_be_sorted:
|
||||
return sorted(list_to_be_sorted, key=itemgetter(property), reverse=ord)
|
||||
|
||||
return list_to_be_sorted
|
||||
|
||||
|
||||
def qrcode_for_url(url):
|
||||
if url:
|
||||
try:
|
||||
qr_code = D4S_QrCode(url)
|
||||
image_path = qr_code.get_qrcode_path()
|
||||
with open(image_path, "rb") as image_file:
|
||||
return base64.b64encode(image_file.read())
|
||||
return ""
|
||||
except Exception as error:
|
||||
log.error("Error on getting qrcode for url: " + url + "error: %s" % error)
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def get_list_of_organizations(limit=10, sort='packages'):
|
||||
to_browse_organizations = []
|
||||
try:
|
||||
data = {}
|
||||
|
||||
if sort:
|
||||
data['sort'] = sort
|
||||
|
||||
data['limit'] = limit
|
||||
data['all_fields'] = True
|
||||
ordered_organizations = []
|
||||
ordered_organizations = logic.get_action('organization_list')({}, data)
|
||||
|
||||
for organization in ordered_organizations:
|
||||
try:
|
||||
to_browse_obj = {}
|
||||
|
||||
if not organization['name']:
|
||||
continue
|
||||
|
||||
to_browse_obj['name'] = organization['name']
|
||||
|
||||
if 'package_count' in organization:
|
||||
to_browse_obj['package_count'] = organization['package_count']
|
||||
|
||||
if 'display_name' in organization:
|
||||
to_browse_obj['display_name'] = organization['display_name']
|
||||
|
||||
image_url = get_url_to_icon_for_ckan_entity(organization['name'], 'organization', False)
|
||||
|
||||
# Using ICON as first option
|
||||
if image_url:
|
||||
to_browse_obj['url'] = image_url
|
||||
# Using object image_url as second one
|
||||
elif 'image_url' in organization and organization['image_url']:
|
||||
to_browse_obj['url'] = organization['image_url']
|
||||
# Default placeholder
|
||||
else:
|
||||
to_browse_obj['url'] = h.url_for('static', filename='images/organisations/icon/placeholder-organization.png')
|
||||
|
||||
to_browse_organizations.append(to_browse_obj)
|
||||
except (logic.NotFound, logic.ValidationError, logic.NotAuthorized) as error:
|
||||
# SILENT
|
||||
log.warn("Error on putting organization: %s" % error)
|
||||
|
||||
log.info("browse %d" % len(ordered_organizations) + " organisation/s")
|
||||
except (logic.NotFound, logic.ValidationError, logic.NotAuthorized) as error:
|
||||
log.error("Error on getting organizations: %s" % error)
|
||||
return []
|
||||
|
||||
return to_browse_organizations
|
||||
|
||||
|
||||
def get_list_of_groups(limit=10, sort='package_count'):
|
||||
to_browse_groups = []
|
||||
try:
|
||||
data = {}
|
||||
if sort:
|
||||
data['sort'] = sort
|
||||
|
||||
data['limit'] = limit
|
||||
data['all_fields'] = True
|
||||
ordered_groups = []
|
||||
ordered_groups = logic.get_action('group_list')({}, data)
|
||||
|
||||
for group in ordered_groups:
|
||||
# print "\n\ngroup %s" %group
|
||||
try:
|
||||
to_browse_obj = {}
|
||||
|
||||
if not group['name']:
|
||||
continue
|
||||
|
||||
to_browse_obj['name'] = group['name']
|
||||
|
||||
if 'package_count' in group:
|
||||
to_browse_obj['package_count'] = group['package_count']
|
||||
|
||||
if 'display_name' in group:
|
||||
to_browse_obj['display_name'] = group['display_name']
|
||||
|
||||
if 'image_url' in group and group['image_url']:
|
||||
to_browse_obj['url'] = group['image_url']
|
||||
else:
|
||||
to_browse_obj['url'] = get_url_to_icon_for_ckan_entity(group['name'], 'group')
|
||||
|
||||
to_browse_groups.append(to_browse_obj)
|
||||
except (logic.NotFound, logic.ValidationError, logic.NotAuthorized) as error:
|
||||
# SILENT
|
||||
log.warn("Error on putting group: %s" % error)
|
||||
|
||||
log.info("browse %d" % len(ordered_groups) + " organisation/s")
|
||||
except (logic.NotFound, logic.ValidationError, logic.NotAuthorized) as error:
|
||||
log.error("Error on getting group: %s" % error)
|
||||
return []
|
||||
|
||||
return to_browse_groups
|
||||
|
||||
|
||||
def get_browse_info_for_organisations_or_groups(type='organization', limit=10, sort_field=None):
|
||||
sort = None
|
||||
if sort_field:
|
||||
sort = sort_field
|
||||
|
||||
if type == 'organization':
|
||||
if sort:
|
||||
return get_list_of_organizations(limit, sort)
|
||||
else:
|
||||
return get_list_of_organizations(limit)
|
||||
|
||||
elif type == 'group':
|
||||
if sort:
|
||||
return get_list_of_groups(limit, sort)
|
||||
else:
|
||||
return get_list_of_groups(limit)
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def get_image_display_for_group(item_id):
|
||||
if item_id:
|
||||
try:
|
||||
item_obj = model.Group.get(item_id)
|
||||
|
||||
if item_obj and item_obj.image_url:
|
||||
return item_obj.image_url
|
||||
else:
|
||||
return h.url_for('static', filename='images/groups/icon/placeholder-group.png')
|
||||
|
||||
except Exception as error:
|
||||
log.error("Error on getting item obj: %s" % item_id + "error: %s" % error)
|
||||
|
||||
|
||||
def get_application_path():
|
||||
if getattr(sys, 'frozen', False):
|
||||
# If the application is run as a bundle, the pyInstaller bootloader
|
||||
# extends the sys module by a flag frozen=True and sets the app
|
||||
# path into variable _MEIPASS'.
|
||||
application_path = sys._MEIPASS
|
||||
else:
|
||||
application_path = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
return application_path
|
||||
|
||||
|
||||
'''
|
||||
Get icon url for input entity type
|
||||
@:param default_placeholder if True returns the URL of default image, otherwise None.
|
||||
'''
|
||||
|
||||
|
||||
def get_url_to_icon_for_ckan_entity(item_name, entity_type=None, default_placeholder=True):
|
||||
if not entity_type or not item_name:
|
||||
return None
|
||||
|
||||
dir_images_full_path = get_application_path() + "/public/images"
|
||||
dir_images_relative_path = "/images"
|
||||
|
||||
if entity_type == 'group':
|
||||
dir_images_full_path += "/groups"
|
||||
dir_images_relative_path += "/groups"
|
||||
placeholder_icon = "placeholder-group.png"
|
||||
elif entity_type == 'organization':
|
||||
dir_images_full_path += "/organisations"
|
||||
dir_images_relative_path += "/organisations"
|
||||
placeholder_icon = "placeholder-organization.png"
|
||||
elif entity_type == 'type':
|
||||
dir_images_full_path += "/types"
|
||||
dir_images_relative_path += "/types"
|
||||
placeholder_icon = "placeholder-type.png"
|
||||
else:
|
||||
return None
|
||||
|
||||
icon_path = os.path.join(dir_images_full_path, "icon", item_name.lower() + ".png")
|
||||
if os.path.isfile(icon_path):
|
||||
return h.url_for('static', filename=os.path.join(dir_images_relative_path, "icon", item_name.lower() + ".png"))
|
||||
elif default_placeholder:
|
||||
return h.url_for('static', filename=os.path.join(dir_images_relative_path, "icon", placeholder_icon))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_user_info(user_id_or_name):
|
||||
if user_id_or_name:
|
||||
try:
|
||||
|
||||
item_obj = model.User.get(user_id_or_name)
|
||||
|
||||
if item_obj:
|
||||
return item_obj
|
||||
|
||||
return None
|
||||
except Exception as error:
|
||||
log.error("Error on getting item obj: %s" % user_id_or_name + "error: %s" % error)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
'''
|
||||
Search the value of my_search_string into input file {ckan_po_file} or the default file ckan.po provided as CKAN language
|
||||
and returns its translate
|
||||
'''
|
||||
|
||||
|
||||
def get_ckan_translate_for(ckan_po_file, my_search_string):
|
||||
my_translate = session.get(TRANSLATE_OF_ + my_search_string)
|
||||
|
||||
if not my_search_string:
|
||||
return ""
|
||||
|
||||
if my_translate:
|
||||
log.info("Translate of '%s' " % my_search_string + " found in session as: %s" % my_translate)
|
||||
return my_translate
|
||||
|
||||
if not ckan_po_file:
|
||||
ckan_po_file = "/usr/lib/ckan/default/src/ckan/ckan/i18n/en_gcube/LC_MESSAGES/ckan.po"
|
||||
|
||||
numlines = 0
|
||||
numfound = 0
|
||||
found = 0
|
||||
line_text = ""
|
||||
|
||||
try:
|
||||
infile = open(ckan_po_file, "r")
|
||||
|
||||
for line in infile:
|
||||
numlines += 1
|
||||
if found > 0:
|
||||
numfound += 1
|
||||
line_text += str(line)
|
||||
found = 0 # reset found
|
||||
|
||||
# found += line.upper().count(my_search_string.upper())
|
||||
found += line.count(my_search_string)
|
||||
|
||||
if found > 0:
|
||||
log.debug("The search string '%s'" % my_search_string + " was found. Read the line: %s" % str(line))
|
||||
|
||||
infile.close()
|
||||
|
||||
except Exception as e:
|
||||
print("Exception during parsing the file %s" % ckan_po_file, e)
|
||||
|
||||
log.info("Recap: '%s' was found" % my_search_string + " %i times " % numfound + "in %i lines" % numlines)
|
||||
log.debug("Line text is: %s" % line_text)
|
||||
|
||||
pattern = '"([A-Za-z0-9_ \./\\-]*)"'
|
||||
m = re.search(pattern, line_text)
|
||||
|
||||
try:
|
||||
my_translate = m.group()
|
||||
except Exception as e:
|
||||
print("Pattern %s" % my_search_string + " not found ", e)
|
||||
|
||||
if my_translate:
|
||||
log.debug("Replacing quotas...")
|
||||
my_translate = my_translate.replace("\"", "")
|
||||
|
||||
log.info("Found the string '%s'" % my_translate + " that translating '%s'" % my_search_string)
|
||||
|
||||
session[TRANSLATE_OF_ + my_search_string] = my_translate
|
||||
session.modified = True
|
||||
#session.save() capire se serve, approccio standard con modified, save forza il salvataggio esplicito della sessione
|
||||
|
||||
return my_translate
|
||||
|
||||
|
||||
def get_location_to_bboxes():
|
||||
config = configparser.ConfigParser()
|
||||
config.optionxform = str
|
||||
location_to_bboxes = {}
|
||||
try:
|
||||
bboxes_file = get_application_path() + "/public/location_to_bboxes.ini"
|
||||
log.debug("bboxes_file is: '%s'" % bboxes_file)
|
||||
config.read(bboxes_file)
|
||||
for section_name in config.sections():
|
||||
log.debug('Location to bboxes Section: ' + section_name)
|
||||
# print ' Options:', parser.options(section_name)
|
||||
for name, value in config.items(section_name):
|
||||
location_to_bboxes[name] = value.replace(",", "%2C")
|
||||
|
||||
ordDictBboxes = collections.OrderedDict(sorted(location_to_bboxes.items()))
|
||||
log.debug("Ordered 'bboxes_file' dict: '%s'" % ordDictBboxes)
|
||||
return ordDictBboxes
|
||||
except Exception as error:
|
||||
log.error("Error on reading file: %s" % bboxes_file + "error: %s" % error)
|
||||
|
||||
def get_content_moderator_system_placeholder():
|
||||
return systemtype_cms_fields_placeholders
|
||||
|
||||
|
||||
#ITemplateHelpers
|
||||
def get_helpers(self):
|
||||
log.info("get_helpers called...")
|
||||
'''Register functions as a template
|
||||
helper function.
|
||||
'''
|
||||
# Template helper function names should begin with the name of the
|
||||
# extension they belong to, to avoid clashing with functions from
|
||||
# other extensions.
|
||||
return {
|
||||
"d4science_hello": d4science_hello,
|
||||
"get_deliverable_type": get_deliverable_type
|
||||
'd4science_theme_get_user_role_for_group_or_org': get_user_role_for_group_or_org,
|
||||
'd4science_theme_get_parents_for_group': get_parents_for_group,
|
||||
'get_header_param': get_header_param,
|
||||
'get_request_param': get_request_param,
|
||||
'get_cookie_value': get_cookie_value,
|
||||
'd4science_theme_markdown_extract_html' : markdown_extract_html,
|
||||
'd4science_theme_get_systemtype_value_from_extras' : get_systemtype_value_from_extras,
|
||||
'd4science_theme_get_systemtype_field_dict_from_session' : get_systemtype_field_dict_from_session,
|
||||
'd4science_theme_get_namespace_separator_from_session' : get_namespace_separator_from_session,
|
||||
'd4science_theme_get_extras' : get_extras,
|
||||
'd4science_theme_count_facet_items_dict' : count_facet_items_dict,
|
||||
'd4science_theme_purge_namespace_to_facet': purge_namespace_to_fieldname,
|
||||
'd4science_get_color_for_type': get_color_for_type,
|
||||
'd4science_get_d4s_namespace_controller': get_d4s_namespace_controller,
|
||||
'd4science_get_extras_indexed_for_namespaces': get_extras_indexed_for_namespaces,
|
||||
'd4science_get_namespaces_dict': get_namespaces_dict,
|
||||
'd4science_get_extra_for_category' : get_extra_for_category,
|
||||
'd4science_get_ordered_dictionary': ordered_dictionary,
|
||||
'd4science_get_qrcode_for_url': qrcode_for_url,
|
||||
'd4science_get_list_of_organizations': get_list_of_organizations,
|
||||
'd4science_get_image_display_for_group': get_image_display_for_group,
|
||||
'd4science_get_list_of_groups': get_list_of_groups,
|
||||
'd4science_get_browse_info_for_organisations_or_groups': get_browse_info_for_organisations_or_groups,
|
||||
'd4science_get_user_info': get_user_info,
|
||||
'd4science_get_url_to_icon_for_ckan_entity' : get_url_to_icon_for_ckan_entity,
|
||||
'd4science_get_ckan_translate_for' : get_ckan_translate_for,
|
||||
'd4science_get_location_to_bboxes' : get_location_to_bboxes,
|
||||
'd4science_get_content_moderator_system_placeholder': get_content_moderator_system_placeholder,
|
||||
}
|
||||
|
||||
def get_deliverable_type():
|
||||
return "This is a custom deliverable type!"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,203 @@
|
|||
import ckan.plugins as plugins
|
||||
import ckan.plugins.toolkit as toolkit
|
||||
import ckan.lib.dictization.model_save as model_save
|
||||
import sqlalchemy as sa
|
||||
from ckan.views.resource import Blueprint
|
||||
from ckanext.d4science import helpers
|
||||
from ckanext.d4science.logic import action, auth
|
||||
from ckanext.d4science.logic import validators
|
||||
from ckan.config.middleware.common_middleware import TrackingMiddleware
|
||||
from ckan.model import Package
|
||||
from logging import getLogger
|
||||
|
||||
from ckan.common import (
|
||||
g
|
||||
)
|
||||
|
||||
d4s_ctg_namespaces_controller = None
|
||||
|
||||
log = getLogger(__name__)
|
||||
|
||||
def remove_check_replicated_custom_key(schema):
|
||||
if schema is not None:
|
||||
schema.pop('__before', None)
|
||||
|
||||
return schema
|
||||
|
||||
def _package_extras_save(extra_dicts, obj, context):
|
||||
''' It can save repeated extras as key-value '''
|
||||
allow_partial_update = context.get("allow_partial_update", False)
|
||||
if extra_dicts is None and allow_partial_update:
|
||||
return
|
||||
|
||||
model = context["model"]
|
||||
session = context["session"]
|
||||
|
||||
log.debug("extra_dicts: "+unicode(str(extra_dicts)).encode('utf-8'))
|
||||
#print "extra_dicts: "+str(extra_dicts)
|
||||
|
||||
extras_list = obj.extras_list
|
||||
#extras = dict((extra.key, extra) for extra in extras_list)
|
||||
old_extras = {}
|
||||
extras = {}
|
||||
for extra in extras_list or []:
|
||||
old_extras.setdefault(extra.key, []).append(extra.value)
|
||||
extras.setdefault(extra.key, []).append(extra)
|
||||
|
||||
#print "old_extras: "+str(old_extras)
|
||||
|
||||
new_extras = {}
|
||||
for extra_dict in extra_dicts or []:
|
||||
#print 'extra_dict key: '+extra_dict["key"] + ', value: '+extra_dict["value"]
|
||||
#new_extras.setdefault(extra_dict["key"], []).append(extra_dict["value"])
|
||||
if extra_dict.get("deleted"):
|
||||
log.debug("extra_dict deleted: "+unicode(extra_dict["key"]).encode('utf-8'))
|
||||
#print 'extra_dict deleted: '+extra_dict["key"]
|
||||
continue
|
||||
|
||||
#if extra_dict['value'] is not None and not extra_dict["value"] == "":
|
||||
if extra_dict['value'] is not None:
|
||||
new_extras.setdefault(extra_dict["key"], []).append(extra_dict["value"])
|
||||
|
||||
log.debug("new_extras: "+unicode(str(new_extras)).encode('utf-8'))
|
||||
#print "new_extras: "+str(new_extras)
|
||||
|
||||
#new
|
||||
for key in set(new_extras.keys()) - set(old_extras.keys()):
|
||||
state = 'active'
|
||||
log.debug("adding key: "+unicode(key).encode('utf-8'))
|
||||
#print "adding key: "+str(key)
|
||||
extra_lst = new_extras[key]
|
||||
for extra in extra_lst:
|
||||
extra = model.PackageExtra(state=state, key=key, value=extra)
|
||||
session.add(extra)
|
||||
extras_list.append(extra)
|
||||
|
||||
#deleted
|
||||
for key in set(old_extras.keys()) - set(new_extras.keys()):
|
||||
log.debug("deleting key: "+unicode(key).encode('utf-8'))
|
||||
#print "deleting key: "+str(key)
|
||||
extra_lst = extras[key]
|
||||
for extra in extra_lst:
|
||||
state = 'deleted'
|
||||
extra.state = state
|
||||
extras_list.remove(extra)
|
||||
|
||||
#changed
|
||||
for key in set(new_extras.keys()) & set(old_extras.keys()):
|
||||
#for each value of new list
|
||||
for value in new_extras[key]:
|
||||
old_occur = old_extras[key].count(value)
|
||||
new_occur = new_extras[key].count(value)
|
||||
log.debug("value: "+unicode(value).encode('utf-8') + ", new_occur: "+unicode(new_occur).encode('utf-8')+ ", old_occur: "+unicode(old_occur).encode('utf-8'))
|
||||
#print "value: "+str(value) + ", new_occur: "+str(new_occur) + ", old_occur: "+str(old_occur)
|
||||
# it is an old value deleted or not
|
||||
if value in old_extras[key]:
|
||||
if old_occur == new_occur:
|
||||
#print "extra - occurrences of: "+str(value) +", are equal into both list"
|
||||
log.debug("extra - occurrences of: "+unicode(value).encode('utf-8') +", are equal into both list")
|
||||
#there is a little bug, this code return always the first element, so I'm fixing with #FIX-STATUS
|
||||
extra_values = get_package_for_value(extras[key], value)
|
||||
#extras_list.append(extra)
|
||||
for extra in extra_values:
|
||||
state = 'active'
|
||||
extra.state = state
|
||||
session.add(extra)
|
||||
#print "extra updated: "+str(extra)
|
||||
log.debug("extra updated: "+unicode(extra).encode('utf-8'))
|
||||
|
||||
elif new_occur > old_occur:
|
||||
#print "extra - a new occurrence of: "+str(value) +", is present into new list, adding it to old list"
|
||||
log.debug("extra - a new occurrence of: "+unicode(value).encode('utf-8') +", is present into new list, adding it to old list")
|
||||
state = 'active'
|
||||
extra = model.PackageExtra(state=state, key=key, value=value)
|
||||
extra.state = state
|
||||
session.add(extra)
|
||||
extras_list.append(extra)
|
||||
old_extras[key].append(value)
|
||||
log.debug("old extra values updated: "+unicode(old_extras[key]).encode('utf-8'))
|
||||
#print "old extra values updated: "+str(old_extras[key])
|
||||
|
||||
else:
|
||||
#remove all occurrences deleted - this code could be optimized, it is run several times but could be performed one shot
|
||||
countDelete = old_occur-new_occur
|
||||
log.debug("extra - occurrence of: "+unicode(value).encode('utf-8')+", is not present into new list, removing "+unicode(countDelete).encode('utf-8')+" occurrence/s from old list")
|
||||
#print "extra - occurrence of: "+str(value) +", is not present into new list, removing "+str(countDelete)+" occurrence/s from old list"
|
||||
extra_values = get_package_for_value(extras[key], value)
|
||||
for idx, extra in enumerate(extra_values):
|
||||
if idx < countDelete:
|
||||
#print "extra - occurrence of: "+str(value) +", is not present into new list, removing it from old list"
|
||||
log.debug("pkg extra deleting: "+unicode(extra.value).encode('utf-8'))
|
||||
#print "pkg extra deleting: "+str(extra.value)
|
||||
state = 'deleted'
|
||||
extra.state = state
|
||||
|
||||
else:
|
||||
#print "pkg extra reactivating: "+str(extra.value)
|
||||
log.debug("pkg extra reactivating: "+unicode(extra.value).encode('utf-8'))
|
||||
state = 'active'
|
||||
extra.state = state
|
||||
session.add(extra)
|
||||
|
||||
else:
|
||||
#print "extra new value: "+str(value)
|
||||
log.debug("extra new value: "+unicode(value).encode('utf-8'))
|
||||
state = 'active'
|
||||
extra = model.PackageExtra(state=state, key=key, value=value)
|
||||
extra.state = state
|
||||
session.add(extra)
|
||||
extras_list.append(extra)
|
||||
|
||||
|
||||
#for each value of old list
|
||||
for value in old_extras[key]:
|
||||
#if value is not present in new list
|
||||
if value not in new_extras[key]:
|
||||
extra_values = get_package_for_value(extras[key], value)
|
||||
for extra in extra_values:
|
||||
#print "not present extra deleting: "+str(extra)
|
||||
log.debug("not present extra deleting: "+unicode(extra).encode('utf-8'))
|
||||
state = 'deleted'
|
||||
extra.state = state
|
||||
|
||||
def get_package_for_value(list_package, value):
|
||||
''' Returns a list of packages containing the value passed in input
|
||||
'''
|
||||
lst = []
|
||||
for x in list_package:
|
||||
if x.value == value:
|
||||
lst.append(x)
|
||||
else:
|
||||
return lst
|
||||
|
||||
return lst
|
||||
|
||||
|
||||
#OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE
|
||||
#gestisce le connessioni al db relazionale utilizzato da ckan
|
||||
def _init_TrackingMiddleware(self, app, config):
|
||||
self.app = app
|
||||
log.debug('TrackingMiddleware d4Science instance')
|
||||
sqlalchemy_url = config.get('sqlalchemy.url')
|
||||
log.debug('sqlalchemy_url read: '+str(sqlalchemy_url))
|
||||
|
||||
sqlalchemy_pool = config.get('sqlalchemy.pool_size')
|
||||
if sqlalchemy_pool is None:
|
||||
sqlalchemy_pool = 5
|
||||
|
||||
log.debug('sqlalchemy_pool read: '+str(sqlalchemy_pool))
|
||||
sqlalchemy_overflow = config.get('sqlalchemy.max_overflow')
|
||||
|
||||
if sqlalchemy_overflow is None:
|
||||
sqlalchemy_overflow = 10;
|
||||
|
||||
log.debug('sqlalchemy_overflow read: '+str(sqlalchemy_overflow))
|
||||
|
||||
try:
|
||||
self.engine = sa.create_engine(sqlalchemy_url, pool_size=int(sqlalchemy_pool), max_overflow=int(sqlalchemy_overflow))
|
||||
except TypeError as e:
|
||||
log.error('pool size does not work: ' +str(e.args))
|
||||
self.engine = sa.create_engine(sqlalchemy_url)
|
||||
|
||||
|
||||
# import ckanext.d4science.cli as cli
|
||||
|
|
@ -57,27 +250,44 @@ class D4SciencePlugin(plugins.SingletonPlugin):
|
|||
# Indica che questo plugin può essere usato come fallback se un tipo specifico non è specificato
|
||||
return False
|
||||
|
||||
#IDatasetForm
|
||||
def package_types(self):
|
||||
# This plugin doesn't handle any special package types, it just
|
||||
# registers itself as the default (above).
|
||||
return []
|
||||
|
||||
def create_package_schema(self):
|
||||
schema = super(D4SciencePlugin, self).create_package_schema()
|
||||
schema.update({
|
||||
'deliverable_type': [ignore_missing, unicode],
|
||||
})
|
||||
schema = remove_check_replicated_custom_key(schema)
|
||||
#schema.update({
|
||||
# 'deliverable_type': [ignore_missing, unicode],
|
||||
#})
|
||||
return schema
|
||||
|
||||
def update_package_schema(self):
|
||||
schema = super(D4SciencePlugin, self).update_package_schema()
|
||||
schema.update({
|
||||
'deliverable_type': [ignore_missing, unicode],
|
||||
})
|
||||
schema = remove_check_replicated_custom_key(schema)
|
||||
#schema.update({
|
||||
# 'deliverable_type': [ignore_missing, unicode],
|
||||
#})
|
||||
return schema
|
||||
|
||||
def show_package_schema(self):
|
||||
schema = super(D4SciencePlugin, self).show_package_schema()
|
||||
schema.update({
|
||||
'deliverable_type': [ignore_missing, unicode],
|
||||
})
|
||||
schema = remove_check_replicated_custom_key(schema)
|
||||
#schema.update({
|
||||
# 'deliverable_type': [ignore_missing, unicode],
|
||||
#})
|
||||
return schema
|
||||
|
||||
#override
|
||||
model_save.package_extras_save = _package_extras_save
|
||||
|
||||
#OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE
|
||||
TrackingMiddleware.__init__ = _init_TrackingMiddleware
|
||||
|
||||
global d4s_ctg_namespaces_controller
|
||||
|
||||
# IBlueprint
|
||||
|
||||
# def get_blueprint(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue