diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..0b344fc Binary files /dev/null and b/.DS_Store differ diff --git a/.project b/.project new file mode 100644 index 0000000..66687f6 --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + ckanext-d4science + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/ckanext/.DS_Store b/ckanext/.DS_Store new file mode 100644 index 0000000..6544702 Binary files /dev/null and b/ckanext/.DS_Store differ diff --git a/ckanext/d4science/.DS_Store b/ckanext/d4science/.DS_Store new file mode 100644 index 0000000..0559bfc Binary files /dev/null and b/ckanext/d4science/.DS_Store differ diff --git a/ckanext/d4science/d4sdiscovery/__init__.py b/ckanext/d4science/d4sdiscovery/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ckanext/d4science/d4sdiscovery/d4s_cache_controller.py b/ckanext/d4science/d4sdiscovery/d4s_cache_controller.py new file mode 100644 index 0000000..2c90297 --- /dev/null +++ b/ckanext/d4science/d4sdiscovery/d4s_cache_controller.py @@ -0,0 +1,102 @@ +import datetime +import logging +import os +import tempfile +import csv + +from .icproxycontroller import NAMESPACE_ID_LABEL + +log = logging.getLogger(__name__) + +CATALINA_HOME = 'CATALINA_HOME' +temp_dir = None +namespaces_dir = None +NAMESPACES_DIR_NAME = "namespaces_for_catalogue" +NAMESPACES_CACHE_FILENAME = "Namespaces_Catalogue_Categories.csv" + + +# D4S_Cache_Controller +class D4S_Cache_Controller(): + namespaces_cache_path = None + __scheduler = None + + def __init__(self): + """ Virtually private constructor. """ + log.debug("__init__ D4S_Cache_Controller") + self._check_cache() + + def _check_cache(self): + + if self.namespaces_cache_path is None: + self.init_temp_dir() + self.namespaces_cache_path = os.path.join(namespaces_dir, NAMESPACES_CACHE_FILENAME) + log.info("The namespaces cache is located at: %s" % self.namespaces_cache_path) + + if not os.path.exists(self.namespaces_cache_path): + log.info("File does not exists creating it") + try: + with open(self.namespaces_cache_path, mode='w') as namespaces_file: + csv.writer(namespaces_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + log.info("Cache created at %s" % self.namespaces_cache_path) + except Exception as e: + print(e) + + ''' Write the list of dictionary with namespaces''' + def write_namespaces(self, namespace_list_of_dict): + # Insert Data + with open(self.namespaces_cache_path, 'w') as namespaces_file: + writer = csv.writer(namespaces_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + writer.writerow([NAMESPACE_ID_LABEL, 'name', 'title', 'description']) + for namespace_dict in namespace_list_of_dict: + #print("namespace %s" % namespace_dict) + writer.writerow([namespace_dict[NAMESPACE_ID_LABEL], namespace_dict['name'], namespace_dict['title'], namespace_dict['description']]) + + log.info("Inserted %d namespaces in the Cache" % len(namespace_list_of_dict)) + + '''Returns the list of dictionary with namespaces''' + def read_namespaces(self): + # Read Data + namespace_list_of_dict = [] + try: + with open(self.namespaces_cache_path, 'r') as namespaces_file: + reader = csv.DictReader(namespaces_file) + for row in reader: + #print("read namespace %s" % row) + namespace_list_of_dict.append(dict(row)) + + log.debug("from Cache returning namespace_list_of_dict %s: " % namespace_list_of_dict) + log.info("from Cache read namespace_list_of_dict with %d item/s " % len(namespace_list_of_dict)) + return namespace_list_of_dict + except Exception as e: + print(e) + + log.info("no namespace in the Cache returning empty list of dict") + return namespace_list_of_dict + + @property + def get_namespaces_cache_path(self): + return self.namespaces_cache_path + + @classmethod + def init_temp_dir(cls): + global temp_dir + global NAMESPACES_DIR_NAME + global namespaces_dir + try: + temp_dir = str(os.environ[CATALINA_HOME]) + temp_dir = os.path.join(temp_dir, "temp") + except KeyError as error: + log.error("No environment variable for: %s" % CATALINA_HOME) + + if temp_dir is None: + temp_dir = tempfile.gettempdir() # using system tmp dir + + log.debug("Temp dir is: %s" % temp_dir) + + namespaces_dir = os.path.join(temp_dir, NAMESPACES_DIR_NAME) + + if not os.path.exists(namespaces_dir): + os.makedirs(namespaces_dir) + + + diff --git a/ckanext/d4science/d4sdiscovery/d4s_extras.py b/ckanext/d4science/d4sdiscovery/d4s_extras.py new file mode 100644 index 0000000..86bfd60 --- /dev/null +++ b/ckanext/d4science/d4sdiscovery/d4s_extras.py @@ -0,0 +1,26 @@ +import logging +log = logging.getLogger(__name__) + +class D4S_Extras(): + + def __init__(self, category_dict={}, extras=[]): + self._category = category_dict + self._extras = extras + + def append_extra(self, k, v): + #print ("self._extras: %s" %self._extras) + if k is not None: + self._extras.append({k:v}) + + @property + def category(self): + return self._category + + @property + def extras(self): + return self._extras + + def __repr__(self): + return 'category: %s'%self._category+' ' \ + 'extras: %s'%self._extras + diff --git a/ckanext/d4science/d4sdiscovery/d4s_namespaces.py b/ckanext/d4science/d4sdiscovery/d4s_namespaces.py new file mode 100644 index 0000000..54f1e90 --- /dev/null +++ b/ckanext/d4science/d4sdiscovery/d4s_namespaces.py @@ -0,0 +1,39 @@ +# Created by Francesco Mangiacrapa +# francesco.mangiacrapa@isti.cnr.it +# ISTI-CNR Pisa (ITALY) + +#OrderedDict([(u'@id', u'extra_information'), (u'name', u'Extra Information'), (u'title', u'Extras'), (u'description', u'This section is about Extra(s)')]), u'contact': OrderedDict([(u'@id', u'contact'), (u'name', u'Contact'), (u'title', u'Contact Title'), (u'description', u'This section is about Contact(s)')]), u'developer_information': OrderedDict([(u'@id', u'developer_information'), (u'name', u'Developer'), (u'title', u'Developer Information'), (u'description', u'This section is about Developer(s)')])} + +import logging +log = logging.getLogger(__name__) + +class D4S_Namespaces(): + + def __init__(self, id=None, name=None, title=None, description=None): + self._id = id + self._name = name + self._title = title + self._description = description + + @property + def id(self): + return self._id + + @property + def name(self): + return self._name + + @property + def title(self): + return self._title + + + @property + def description(self): + return self._description + + def __repr__(self): + return '{id: %s'%self.id+', ' \ + 'name: %s'%self.name+ ', ' \ + 'title: %s'%self.title+ ', ' \ + 'description: %s'%self.description+ '}' diff --git a/ckanext/d4science/d4sdiscovery/d4s_namespaces_controller.py b/ckanext/d4science/d4sdiscovery/d4s_namespaces_controller.py new file mode 100644 index 0000000..b187908 --- /dev/null +++ b/ckanext/d4science/d4sdiscovery/d4s_namespaces_controller.py @@ -0,0 +1,125 @@ +import logging +import time + +from .d4s_cache_controller import D4S_Cache_Controller +from .icproxycontroller import D4S_IS_DiscoveryCatalogueNamespaces +from threading import Event, Thread + +CATEGORY = 'category' +NOCATEOGORY = 'nocategory' + +log = logging.getLogger(__name__) + +cancel_future_calls = None + +# Refreshing time for namespaces cache in secs. +NAMESPACES_CACHE_REFRESHING_TIME = 60 * 60 + + +# Funtion to call repeatedly another function +def call_repeatedly(interval, func, *args): + log.info("call_repeatedly called on func '{}' with interval {} sec".format(func.__name__, interval)) + stopped = Event() + + def loop(): + while not stopped.wait(interval): # the first call is in `interval` secs + func(*args) + + th = Thread(name='daemon_caching_namespaces', target=loop) + th.setDaemon(True) + th.start() + return stopped.set + + +def reload_namespaces_from_IS(urlICProxy, resourceID, gcubeToken): + log.info("_reload_namespaces_from_IS called") + try: + discovery_ctg_namespaces = D4S_IS_DiscoveryCatalogueNamespaces(urlICProxy, resourceID, gcubeToken) + namespaces_list_of_dict = discovery_ctg_namespaces.getNamespacesDictFromResource() + + if namespaces_list_of_dict is not None and len(namespaces_list_of_dict) > 0: + log.debug("namespaces read from IS are: %s" % namespaces_list_of_dict) + D4S_Cache_Controller().write_namespaces(namespaces_list_of_dict) + else: + log.info("namespaces list read from IS is empty. Skipping caching update") + + except Exception as e: + log.error("Error occurred on reading namespaces from IS and refilling the cache!") + log.error(e) + + +# D4S_IS_DiscoveryCatalogueNamespacesController is used to discovery namespaces for Catalogue Categories (implemented as a Singleton) +# @param: urlICProxy is the URI of IC proxy rest-full service provided by IS +# @param: resourceID is the resource ID of the Generic Resource: "Namespaces Catalogue Categories" +# @param: gcubeToken the gcube token used to contact the IC proxy +class D4S_Namespaces_Controller(): + __instance = None + + @staticmethod + def getInstance(): + """ Static access method. """ + if D4S_Namespaces_Controller.__instance is None: + D4S_Namespaces_Controller() + + return D4S_Namespaces_Controller.__instance + + def __init__(self): + """ Virtually private constructor. """ + log.debug("__init__ D4S_Namespaces_Controller") + + if D4S_Namespaces_Controller.__instance is not None: + raise Exception("This class is a singleton!") + else: + D4S_Namespaces_Controller.__instance = self + + self._d4s_cache_controller = D4S_Cache_Controller() + self._urlICProxy = None + self._resourceID = None + self._gcubeToken = None + + def load_namespaces(self, urlICProxy, resourceID, gcubeToken): + log.debug("readNamespaces called") + self._urlICProxy = urlICProxy + self._resourceID = resourceID + self._gcubeToken = gcubeToken + return self._check_namespaces() + + def _read_namespaces(self): + return self._d4s_cache_controller.read_namespaces() + + def _check_namespaces(self): + log.debug("_check_namespaces called") + + if self._d4s_cache_controller is None: + self._d4s_cache_controller = D4S_Cache_Controller() + + namespace_list = self._read_namespaces() + + # when the Cache is empty + if namespace_list is None or not namespace_list: + # reading namespaces from IS and filling the DB + log.info("The Cache is empty. Reading the namespace from IS and filling the Cache") + reload_namespaces_from_IS(self._urlICProxy, self._resourceID, self._gcubeToken) + # reloading the namespaces from the cache + namespace_list = self._read_namespaces() + + # starting Thread daemon for refreshing the namespaces Cache + global cancel_future_calls + if cancel_future_calls is None: + cancel_future_calls = call_repeatedly(NAMESPACES_CACHE_REFRESHING_TIME, reload_namespaces_from_IS, + self._urlICProxy, + self._resourceID, + self._gcubeToken) + + return namespace_list + + def get_dict_ctg_namespaces(self): + log.debug("get_dict_ctg_namespaces called") + namespace_list_of_dict = self._check_namespaces() + return self.convert_namespaces_to_d4s_namespacedict(namespace_list_of_dict) + + # Private method + @staticmethod + def convert_namespaces_to_d4s_namespacedict(namespace_list_of_dict): + log.debug("convert_namespaces_to_d4s_namespacedict called on %s" % namespace_list_of_dict) + return D4S_IS_DiscoveryCatalogueNamespaces.to_namespaces_dict_index_for_id(namespace_list_of_dict) diff --git a/ckanext/d4science/d4sdiscovery/d4s_namespaces_extras_util.py b/ckanext/d4science/d4sdiscovery/d4s_namespaces_extras_util.py new file mode 100644 index 0000000..3073528 --- /dev/null +++ b/ckanext/d4science/d4sdiscovery/d4s_namespaces_extras_util.py @@ -0,0 +1,85 @@ +import logging +import collections +from collections import OrderedDict +from .d4s_extras import D4S_Extras + +CATEGORY = 'category' +NOCATEOGORY = 'nocategory' + +log = logging.getLogger(__name__) + +# D4S_Namespaces_Extra_Util is used to get the extra fields indexed for D4Science namespaces +# @param: namespace_dict is the namespace dict of D4Science namespaces (defined in the Generic Resource: "Namespaces Catalogue Categories") +# @param: extras is the dictionary of extra fields for a certain item +class D4S_Namespaces_Extra_Util(): + + def get_extras_indexed_for_namespaces(self, namespace_dict, extras): + extras_for_categories = collections.OrderedDict() + + # ADDING ALL EXTRAS WITH NAMESPACE + for namespaceid in list(namespace_dict.keys()): + dict_extras = None + nms = namespaceid + ":" + #has_namespace_ref = None + for key, value in extras.items(): + k = key + v = value + # print "key: " + k + # print "value: " + v + if k.startswith(nms): + + if namespaceid not in extras_for_categories: + extras_for_categories[namespaceid] = collections.OrderedDict() + + dict_extras = extras_for_categories[namespaceid] + log.debug("dict_extras %s "%dict_extras) + + if (dict_extras is None) or (not dict_extras): + dict_extras = D4S_Extras(namespace_dict.get(namespaceid), []) + log.debug("dict_extras after init %s " % dict_extras) + + #print ("dict_extras after init %s " % dict_extras) + log.debug("replacing namespace into key %s " % k +" with empty string") + nms = namespaceid + ":" + k = k.replace(nms, "") + dict_extras.append_extra(k, v) + extras_for_categories[namespaceid] = dict_extras + log.debug("adding d4s_extra: %s " % dict_extras+ " - to namespace id: %s" %namespaceid) + #has_namespace_ref = True + #break + + #ADDING ALL EXTRAS WITHOUT NAMESPACE + for key, value in extras.items(): + k = key + v = value + + has_namespace_ref = None + for namespaceid in list(namespace_dict.keys()): + nms = namespaceid + ":" + #IF KEY NOT STARTING WITH NAMESPACE + if k.startswith(nms): + has_namespace_ref = True + log.debug("key: %s " % k + " - have namespace: %s" % nms) + break + + if has_namespace_ref is None: + log.debug("key: %s " % k + " - have not namespace") + if NOCATEOGORY not in extras_for_categories: + extras_for_categories[NOCATEOGORY] = collections.OrderedDict() + + dict_extras_no_cat = extras_for_categories[NOCATEOGORY] + #print ("dict_extras_no_cat %s " % dict_extras_no_cat) + + if (dict_extras_no_cat is None) or (not dict_extras_no_cat): + dict_extras_no_cat = D4S_Extras(NOCATEOGORY, []) + + #print ("adding key: %s "%k+" - value: %s"%v) + log.debug("NOCATEOGORY adding key: %s " % k + " - value: %s" % v) + + dict_extras_no_cat.append_extra(k, v) + log.debug("dict_extras_no_cat %s " % dict_extras_no_cat) + extras_for_categories[NOCATEOGORY] = dict_extras_no_cat + log.debug("extras_for_categories NOCATEOGORY %s " % extras_for_categories) + + return extras_for_categories + diff --git a/ckanext/d4science/d4sdiscovery/icproxycontroller.py b/ckanext/d4science/d4sdiscovery/icproxycontroller.py new file mode 100644 index 0000000..3c2ce9b --- /dev/null +++ b/ckanext/d4science/d4sdiscovery/icproxycontroller.py @@ -0,0 +1,107 @@ +import logging +import urllib.request, urllib.error, urllib.parse +from lxml import etree + +import xmltodict +import collections + +from .d4s_namespaces import D4S_Namespaces + +XPATH_NAMESPACES = "/Resource/Profile/Body/namespaces" +gcubeTokenParam = "gcube-token" +NAMESPACE_ID_LABEL = '@id' + +log = logging.getLogger(__name__) + +def getResponseBody(uri): + req = urllib.request.Request(uri) + try: + resp = urllib.request.urlopen(req, timeout=20) + except urllib.error.HTTPError as e: + log.error("Error on contacting URI: %s" % uri) + log.error("HTTPError: %d" % e.code) + return None + except urllib.error.URLError as e: + # Not an HTTP-specific error (e.g. connection refused) + log.error("URLError - Input URI: %s " % uri + " is not valid!!") + return None + else: + # 200 + body = resp.read() + return body + + +# D4S_IS_DiscoveryCatalogueNamespaces is used to discovery namespaces for Catalogue Categories. +# @param: urlICProxy is the URI of IC proxy rest-full service provided by IS +# @param: resourceID is the resource ID of the Generic Resource: "Namespaces Catalogue Categories" +# @param: gcubeToken the gcube token used to contact the IC proxy +class D4S_IS_DiscoveryCatalogueNamespaces(): + + def __init__(self, urlICProxy, resourceID, gcubeToken): + if not isinstance(urlICProxy, str): + raise ValueError("urlICProxy must be a string") + if not isinstance(resourceID, str): + raise ValueError("resourceID must be a string") + if not isinstance(gcubeToken, str): + raise ValueError("gcubeToken must be a string") + self.urlICProxy = urlICProxy + self.resourceID = resourceID + self.gcubeToken = gcubeToken + + def getNamespacesDictFromResource(self): + + doc = {} + namespace_list = [] + + try: + + uri = self.urlICProxy + "/" + self.resourceID + "?" + gcubeTokenParam + "=" + self.gcubeToken + log.info("Contacting URL: %s" % uri) + theResource = getResponseBody(uri) + log.debug("Resource returned %s " % theResource) + theResourceXML = etree.XML(theResource) + theNamespaces = theResourceXML.xpath(XPATH_NAMESPACES) + log.debug("The body %s" % etree.tostring(theNamespaces[0])) + + if theNamespaces is not None and theNamespaces[0] is not None: + bodyToString = etree.tostring(theNamespaces[0]) + doc = xmltodict.parse(bodyToString) + else: + log.warning("No Namespace for Catalogue Categories found, returning None") + except Exception as inst: + log.error("Error on getting catalogue namespaces: " + str(inst)) + log.info("Returning empty list of namespaces") + return namespace_list + + log.debug("IS namespaces resource to dict is: %s" % doc) + + + if ('namespaces' in doc): + # log.debug('Namespaces obj %s:' % doc['namespaces']) + namespaces = doc['namespaces'] + if doc is not None and 'namespace' in namespaces: + namespace_list = namespaces['namespace'] + + log.info("Loaded %d namespaces from IS resource" % len(namespace_list)) + return namespace_list + + @staticmethod + def to_namespaces_dict_index_for_id(namespace_list): + namespace_dict = collections.OrderedDict() + log.debug("namespaces to dict: %s" % namespace_list) + try: + if namespace_list is not None and len(namespace_list) > 0: + for namespace in namespace_list: + try: + if NAMESPACE_ID_LABEL in namespace: + namespace_dict[namespace[NAMESPACE_ID_LABEL]] = D4S_Namespaces( + namespace[NAMESPACE_ID_LABEL], + namespace['name'], + namespace['title'], + namespace['description']) + except Exception as inst: + log.error("Error on converting catalogue namespaces: " + str(inst)) + except Exception as inst: + log.error("Error on checking namespace_list: " + str(inst)) + # print "namespace_dict to Nam: %s"%namespace_dict + return namespace_dict diff --git a/ckanext/d4science/fanstatic/.gitignore b/ckanext/d4science/fanstatic/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/ckanext/d4science/fanstatic/d4science_scripts.js b/ckanext/d4science/fanstatic/d4science_scripts.js new file mode 100644 index 0000000..eb44b31 --- /dev/null +++ b/ckanext/d4science/fanstatic/d4science_scripts.js @@ -0,0 +1,305 @@ +/* ===================================================== + JavaScript used by CKAN plugin: 'd4science_theme' + Created by Francesco Mangiacrapa ISTI-CNR Pisa, Italy + ===================================================== */ + +/* +questa pagina incapsula ckan in un iframe della pagina del sito d4science??? +*/ +CKAN_D4S_Breadcrumb_Manager = { + + breadcrumbShow : function (show) { + + var breadcrumb = document.getElementById('ckan-breadcrumb'); + console.log('breadcrumb is '+breadcrumb) + if(breadcrumb){ + if(show){ + breadcrumb.style.display = 'block'; + this.organizationTreeShow(true); + } else{ + breadcrumb.style.display = 'none'; + this.organizationTreeShow(false); + } + } + + //var elements = document.getElementsByTagName('a'); + //for(var i = 0, len = elements.length; i < len; i++) { + // elements[i].onclick = function () { + // //alert("You clicked an external link to: " + this.href); + // //window.parent.add_hide_breadcrumb_to_dom(false); + // this.add_hide_breadcrumb_to_dom(false); + // } + //} + }, + + organizationTreeShow : function (show) { + var trees = document.getElementsByClassName("hierarchy-tree-top"); + + if (trees){ + for (i = 0; i < trees.length; i++) { + if(show){ + trees[i].style.display = 'block'; + } else{ + trees[i].style.display = 'none'; + } + } + } + }, + + checkBreadcrumbShow : function () { + + var showBdc = this.getSessionStorageItem("showbreadcrumb") + //console.log("showBdc is: "+showBdc) + if(showBdc != undefined && showBdc=="false"){ + console.log("Show breadcrumb false"); + this.breadcrumbShow(false); + }else{ + console.log("Show breadcrumb true"); + this.breadcrumbShow(true); + } + }, + + + setSessionStorageItem : function (item_key, item_value) { + + // Check browser support + if (typeof(Storage) !== "undefined") { + // Store + sessionStorage.setItem(item_key, item_value); + return true; + } else { + console.log("Sorry, your browser does not support Web Storage..."); + return false; + } + }, + + + getSessionStorageItem : function (item_key) { + + // Check browser support + if (typeof(Storage) !== "undefined") { + // Store + return sessionStorage.getItem(item_key); + } else { + console.log("Sorry, your browser does not support Web Storage..."); + return undefined; + } + } + +} + + +CKAN_D4S_Functions_Util = { + + getPosition : function(canvas, event){ + var x = new Number(); + var y = new Number(); + try { + if (event.clientX != undefined && event.clientY != undefined) + { + + x = event.clientX; + y = event.clientY; + } + else // Firefox method to get the position + { + x = event.clientX + document.body.scrollLeft + + document.documentElement.scrollLeft; + y = event.clientY + document.body.scrollTop + + document.documentElement.scrollTop; + } + x -= canvas.offsetLeft; + y -= canvas.offsetTop; + }catch (err) { + //silent error + } + return '{"posX": "'+x+'", "posY": "'+y+'"}'; + }, + + // When the user clicks on div, open the popup + showPopupD4S : function(event, my_div, my_popup_left_position) { + var popup = document.getElementById(my_div); + var clickPosition = this.getPosition(my_div, event) + var myPosition = JSON.parse(clickPosition); + this.closePopups(my_div); + // When the user clicks anywhere outside of the modal, close it + /*window.onclick = function(event) { + if (event.target != popup) { + popup.style.display = "none"; + } + }*/ + popup.classList.toggle("show"); + + if(my_popup_left_position){ + popup.style.left = my_popup_left_position; + } + else if (myPosition.posX){ + popup.style.left = myPosition.posX + "px"; + } + }, + + closePopups : function ($target) { + var popups = document.getElementsByClassName('popuptext'); + for (i = 0; i < popups.length; i++) { + if (popups[i].getAttribute('id') != $target) { + popups[i].classList.remove('show'); + } + } + }, + + checkURL : function (url) { + //console.log('checking url: '+url) + var regex = new RegExp('^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/|www\.|ftp:\/\/)+[^ "]+$'); + if (regex.test(url)) { + return true; + } + return false; + } + +} + +CKAN_D4S_HTMLMessage_Util = { + + postHeightToPortlet : function (selectedProduct, product) { + var h = document.body.scrollHeight + "px"; + var p = ""; + var msg = ""; + //WORK AROUND IF TWO MESSAGES ARE SENT FROM A PAGE OF A PRODUCT + //THE MESSAGE WITH 'NULL' PRODUCT IS SKIPPED + //console.log("window.location.pathname? "+window.location.pathname); + var pathArray = window.location.pathname.split('/'); + var productContext = "dataset"; + if(pathArray.length>1){ + //console.log("pathArray is: "+pathArray); + var secondLevelLocation = pathArray[1]; //it is the second level location + //console.log("secondLevelLocation is: "+secondLevelLocation); + //console.log("h is: "+h); + if(secondLevelLocation == productContext){ //is it product context? + if(product !== 'undefined' && product !== null){ + p = product; + //console.log("product selected is: "+p); + }else{ + //console.log("product is null or undefined, passing only height"); + msg = "{\"height\": \""+h+"\"}"; + //window.postMessage(msg,'*'); + this.postMessageToParentWindow(msg); + return; + } + } + } + + //msg = "{'height': '"+h+"', 'product': '"+p+"'}"; + msg = "{\"height\": \""+h+"\", \"product\": \""+p+"\"}"; + //window.postMessage(msg,'*'); + //console.log("posting message in the window: "+msg); + this.postMessageToParentWindow(msg); + }, + + postMessageToParentWindow : function (msg) { + //window.postMessage(msg,'*'); + //console.log("posting message in the window: "+msg); + if(window.parent!=null){ + console.log("posting message in the parent window: "+msg); + window.parent.postMessage(msg,'*'); + } + return; + } + +} + +CKAN_D4S_JSON_Util = { + + + //ADDED by Francesco Mangiacrapa + appendHTMLToElement : function(containerID, elementHTML){ + + var divContainer = document.getElementById(containerID); + divContainer.innerHTML = ""; + divContainer.appendChild(elementHTML); + }, + + //ADDED by Francesco Mangiacrapa + jsonToHTML : function(containerID, cssClassToTable) { + + try + { + var jsonTxt = document.getElementById(containerID).innerHTML; + var jsonObj = JSON.parse(jsonTxt); + //console.log(jsonObj.length) + + if(jsonObj.length==undefined) + jsonObj = [jsonObj] + //console.log(jsonObj.length) + + // EXTRACT VALUE FOR HTML HEADER. + var col = []; + for (var i = 0; i < jsonObj.length; i++) { + for (var key in jsonObj[i]) { + //console.log('key json' +key) + if (col.indexOf(key) === -1) { + col.push(key); + } + } + } + + // CREATE DYNAMIC TABLE. + var table = document.createElement("table"); + var addDefaultCss = "json-to-html-table-column"; + if(cssClassToTable){ + addDefaultCss = cssClassToTable; + } + try{ + table.classList.add(addDefaultCss); + }catch(e){ + console.log('invalid css add', e); + } + + // ADD JSON DATA TO THE TABLE AS ROWS. + for (var i = 0; i < col.length; i++) { + + tr = table.insertRow(-1); + var firstCell = tr.insertCell(-1); + //firstCell.style.cssText="font-weight: bold; text-align: center; vertical-align: middle;"; + firstCell.innerHTML = col[i]; + for (var j = 0; j < jsonObj.length; j++) { + var tabCell = tr.insertCell(-1); + var theValue = jsonObj[j][col[i]]; + /* console.log(theValue + ' is url? '+isUrl);*/ + if(CKAN_D4S_Functions_Util.checkURL(theValue)){ + theValue = ''+theValue+''; + } + + tabCell.innerHTML = theValue; + } + } + + // FINALLY ADD THE NEWLY CREATED TABLE WITH JSON DATA TO A CONTAINER. + this.appendHTMLToElement(containerID, table); + + } + catch(e){ + console.log('invalid json', e); + } + } +} + + +//Task #8032 +window.addEventListener("message", + function (e) { + + var curr_loc = window.location.toString() + var orgin = e.origin.toString() + if(curr_loc.startsWith(orgin)){ + //alert("ignoring message from myself"); + return; + } + //console.log("origin: "+e.data) + if(e.data == null) + return; + + var pMess = JSON.parse(e.data) + //console.log(pMess.explore_vres_landing_page) + window.linktogateway = pMess.explore_vres_landing_page; + },false); + \ No newline at end of file diff --git a/ckanext/d4science/fanstatic/d4science_theme.css b/ckanext/d4science/fanstatic/d4science_theme.css new file mode 100644 index 0000000..9e416b1 --- /dev/null +++ b/ckanext/d4science/fanstatic/d4science_theme.css @@ -0,0 +1,844 @@ +/* ===================================================== + The "account masthead" bar across the top of the site + ===================================================== */ + +.account-masthead { + background-color: #ccc; +} +/* The "bubble" containing the number of new notifications. */ +.account-masthead .account .notifications a span { + background-color: #9fa0a2; +} +/* The text and icons in the user account info. */ +.account-masthead .account ul li a { + color: rgba(255, 255, 255, 0.6); +} +/* The user account info text and icons, when the user's pointer is hovering + over them. */ +.account-masthead .account ul li a:hover { + color: rgba(255, 255, 255, 0.7); +/* background-color: black;*/ +} + + +/* ======================================================================== + The main masthead bar that contains the site logo, nav links, and search + ======================================================================== */ + +.masthead { + background: #eee url("/bg-noise.png") repeat scroll 0 0; + border-top: 1px solid #555; + padding-top: 5px; + padding-bottom: 15px !important; + border-bottom: 1px solid #999; +/* background-image: url("/bg-pattern.min.svg") !important; */ +} + +.masthead .navigation .nav-pills li a{ + color: #187794; +} + +a.logo > img{ + margin-bottom: 5px; +} + +/* The "navigation pills" in the masthead (the links to Datasets, + Organizations, etc) when the user's pointer hovers over them. */ +.masthead .navigation .nav-pills li a:hover { +/* background-color: rgb(48, 48, 48);*/ + color: white; +} +/* The "active" navigation pill (for example, when you're on the /dataset page + the "Datasets" link is active). */ +.masthead .navigation .nav-pills li.active a { + background-color: #d2d2d5; +} +/* The "box shadow" effect that appears around the search box when it + has the keyboard cursor's focus. */ +.masthead input[type="text"]:focus { + -webkit-box-shadow: inset 0px 0px 2px 0px rgba(0, 0, 0, 0.7); + box-shadow: inset 0px 0px 2px 0px rgba(0, 0, 0, 0.7); +} + + +/* =========================================== + The content in the middle of the front page + =========================================== */ + +/* Remove the "box shadow" effect around various boxes on the page. */ +.box { + box-shadow: none; +} +.hero { + background: #FEFEFE repeat scroll 0 0 !important; +} +/* Remove the borders around the "Welcome to CKAN" and "Search Your Data" + boxes. */ +.hero .box { + /*border: none;*/ + margin-top: 10px !important; +} +/* Change the colors of the "Search Your Data" box. */ +.homepage .module-search .module-content { + color: rgb(68, 68, 68); + background-color: white; +} +/* Change the background color of the "Popular Tags" box. */ +.homepage .module-search .tags { + background-color: #fcfcfc; +} + +.homepage-title{ + font-size: 20px; + font-weight: bold; + color: #202020; + margin-bottom: 20px; +} + +/* Change the background color of the "Popular Tags" box. */ +.homepage .module-search h3{ + color: #444; +} + +/* Remove some padding. This makes the bottom edges of the "Welcome to CKAN" + and "Search Your Data" boxes line up. */ +.module-content:last-child { + /*padding-bottom: 0px;*/ +} +.homepage .module-search { + padding: 0px; +} +/* Add a border line between the top and bottom halves of the front page. */ +.homepage [role="main"] { + border-bottom: 1px solid #bbb; + padding: 10px 0; +} + +.homepage .stats ul li a b{ + font-size: 30px !important; +} + +[role="main"], .main { +/* background: #f5f6fa url("/bg-pattern.min.svg") repeat; scroll 0 0;*/ + /*background: #fafafa url("/bg-pattern.svg") repeat; scroll 0 0;*/ + background: #fdfdfd none repeat scroll 0 0; + min-height: 0px !important; +} + +.media-item-homepage { + background-color: white; + border-radius: 3px; + float: left; + margin: 15px 0 0 15px; + overflow: hidden; + padding-left: 10px; + padding-right: 10px; + position: relative; + text-align: center; + width: 150px; +} + +.media-heading-homepage { + font-size: 16px; + hyphens: auto; + line-height: 1.3; + margin: 5px 0; +} + +.media-grid-homepage { + -moz-border-bottom-colors: none; + -moz-border-left-colors: none; + -moz-border-right-colors: none; + -moz-border-top-colors: none; +/* background: #fbfbfb url("../../../base/images/bg.png") repeat scroll 0 0; + border-color: #dddddd; + border-image: none; + border-style: solid; + border-width: 1px 0;*/ + list-style: outside none none; + margin: 0 -10px; + padding-bottom: 15px; +} +.media-grid-homepage::before, .media-grid::after { + content: ""; + display: table; + line-height: 0; +} +.media-grid-homepage::after { + clear: both; +} + +.background-circle{ + padding: 10px 10px; + display: inline-block !important; + -webkit-border-radius: 90px; + -moz-border-radius: 90px; + border-radius: 90px; + background-color: #4679b2; + text-decoration: none !important; +} + +.color-white{ + color: white !important; +} + +.badge-circle { + border-radius: 50% 50% 50% 50% !important; + height: 60px; + text-align: center; + vertical-align: middle; + width: 65px; + background-color: #4679b2; + display: inline-block !important; + padding-top: 5px; + text-decoration: none !important; +} + +/* ==================================== + The footer at the bottom of the site + ==================================== */ + +.site-footer, body { + background-color: #bbb; + font-family: "Lato","Helvetica Neue",Helvetica,Arial,sans-serif; + font-size: 16px; +} +/* The text in the footer. */ +.site-footer, +.site-footer label, +.site-footer small { + color: rgba(255, 255, 255, 0.6); +} +/* The link texts in the footer. */ +.site-footer a { + color: rgba(255, 255, 255, 0.6); +} + +.site-footer-internal{ + min-height: 10px; + padding: 2px 0; + font-size: 12px; +} + +.site-footer-internal { + /*background-color: rgba(255, 255, 255, 0.6);*/ + text-align: center; + /*display: inline-block;*/ +} + +.site-footer-internal, +.site-footer-internal label, +.site-footer-internal small { + +} + +.site-footer-internal a { + display: inline-block; +} + +.d4s-hide-text { + background-color: transparent; + border: 0 none; + color: transparent; + font: 0px/0 a; + text-shadow: none; +} + +.d4science-footer-logo { + background: url("/gCube_70.png") no-repeat scroll left top rgba(0, 0, 0, 0); + height: 32px; + margin-top: 2px; + text-indent: -900em; + width: 75px; +} + +.d4s-ckan-footer-logo { + background: rgba(0, 0, 0, 0) url("/ckan-logo-footer.png") no-repeat scroll left top; + height: 21px; + margin-top: 2px; + text-indent: -900em; + width: 69px; +} + +.site-footer-d4science { + font-size: 14px; + color: #f5f5f5; + text-align: center; + height: 25px; + padding-top: 5px; + background-color: #7F7F7F; +} + +.site-footer-d4science a { + font-weight: bold; + text-decoration: none; + color: white; +} + + +/* ==================================== + Base elements of the site + ==================================== */ + +div .principaltitle { + color: inherit; + font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; + font-size: 20px; + font-weight: bold; + line-height: 1.2; + margin: 15px 0; + text-rendering: optimizelegibility; + word-break: break-all; + padding-bottom: 10px; + padding-top: 5px; + border-bottom: 1px solid #eee; +} + +div .notes { + color: #444444; + font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; + font-size: 14px; + line-height: 1.3; + text-align: justify; + word-break: break-all; +} + +div .infotitle { + font-size: 15px; + hyphens: auto; + line-height: 1.3; + word-break: break-all; + font-weight: bold; +} + +.toolbar .breadcrumb{ + font-size: 16px !important; +} + +.box{ + border: 0px !important; +} + +div .sectiontitle{ + color: #9F9F9F; + font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; + font-size: 17px; + font-weight: bold; + margin: 20px 0; + margin-top: 20px; + margin-bottom: 10px; + text-rendering: optimizelegibility; +} + +section .well { + background-color: #fdfdfd !important; + border: 1px solid #e3e3e3; + border-radius: 4px; + box-shadow: none !important; + margin-bottom: 20px; + min-height: 20px; + +} + +.page-heading { + font-size: 18px; + line-height: 1.2; + margin-top: 20px; + margin-bottom: 0px; +} + +#dataset-resources .resource-list{ + background-color: #fdfdfd !important; + border: 1px solid #e3e3e3; + border-radius: 4px; + box-shadow: none !important; + margin: -1px 0 !important; +} + +.wrapper{ + border: 1px solid #d0d0d0; + box-shadow: 0 0 0 2px rgba(0, 0, 0, 0.05); + border-radius: 3px +} + +.home-popular{ + padding-top: 25px; +} + +.logo-homepage{ + max-height: 60px; +} + +.statistics-show{ + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-radius: 5px; + color: #444444; + text-decoration: none; +} + +.d4s-center-cropped{ + text-align: center; + background-color: #eee; + border: 1px solid #ddd; + padding-bottom: 10px; + padding-top: 10px; +} + +.tag-list { + font-size: 14px; +} + + +/* ==================================== + Acquired Dataset + ==================================== */ +.label-acquired { + background-color: #55a1ce; +} + +.label-owner { + background-color: #e0051e; +} + +.divider { + margin-left:10px; + height:auto; + display:inline-block; +} + +/* ==================================== + List Dataset + ==================================== */ + +/*LEFT +.show_meatadatatype { + color: white; + display: inline-block; // Inline elements with width and height. TL;DR they make the icon buttons stack from left-to-right instead of top-to-bottom + position: relative; // All 'absolute'ly positioned elements are relative to this one + margin-bottom: 20px; + margin-left: 25px; +} +*/ + +/*RIGHT*/ +.show_meatadatatype { + color: white; + display: inline-block; + float: right; + margin-right: 2px; + margin-top: -20px; + position: relative; +} + + + +/* LEFT + * Position the badge within the relatively positioned button +.button__badge { + background-color: #fa3e3e; + border-radius: 2px; + color: white; + + padding: 1px 6px; + font-size: 10px; + + position: absolute; + top: 0; + right: 0; +}*/ + + + +/* RIGTH */ +.button__badge { + color: #808080; + padding: 0px 2px; + font-size: 10px; + top: 0; + right: 0; + font-family: sans-serif, times, georgia; +} + +/* ==================================== + Modal Popup + ==================================== */ + +/* Popup container - can be anything you want */ +/* The Modal (background) */ +.d4s_modal { + display: none; /* Hidden by default */ + position: fixed; /* Stay in place */ + z-index: 10001; /* Sit on top (NB. At 1000 there is the zoom in/out of the Map Widget)*/ + /*padding-top: 100px;*/ /* Location of the box */ + left: 0; + top: 0; + width: 100%; /* Full width */ + height: 100%; /* Full height */ + overflow: auto; /* Enable scroll if needed */ + background-color: rgb(0,0,0); /* Fallback color */ + background-color: rgba(0,0,0,0.4); /* Black w/ opacity */ +} + +/* Modal Content */ +.d4s_modal-content { + background-color: #fefefe; + /*margin: auto;*/ + padding: 20px; + border: 1px solid #888; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-radius: 5px; + position: absolute; + left: 50%; + margin-left: -225px; + width: 450px; +} + +/* The Close Button */ +.d4s_close { + color: #aaaaaa; + float: right; + font-size: 28px; + font-weight: bold; + padding-left: 20px; +} + +.d4s_close:hover, +.d4s_close:focus { + color: #000; + text-decoration: none; + cursor: pointer; +} + +.d4s_div_clickable{ + cursor: pointer; +} + +/*==================================== +D4S POPUP +======================================*/ + +/* Popup container - can be anything you want */ +.popupD4SNoArrow { + position: relative; + cursor: pointer; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +/* The actual popup */ +.popupD4SNoArrow .popuptext { + visibility: hidden; + width: 300px; + background-color: #555; + color: #fff; + text-align: center; + border-radius: 6px; + padding: 8px; + position: absolute; + z-index: 1; + bottom: 125%; + left: 50%; + margin-left: -150px; +} + +/* Toggle this class - hide and show the popup */ +.popupD4SNoArrow .show { + visibility: visible; + -webkit-animation: fadeIn 1s; + animation: fadeIn 1s; +} + + +/* Popup container - can be anything you want */ +.popupD4S { + position: relative; + cursor: pointer; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +/* The actual popup */ +.popupD4S .popuptext { + visibility: hidden; + width: 300px; + background-color: #555; + color: #fff; + text-align: center; + border-radius: 6px; + padding: 8px; + position: absolute; + z-index: 1; + bottom: 125%; + left: 50%; + margin-left: -150px; +} + +/* Popup arrow */ +.popupD4S .popuptext::after { + content: ""; + position: absolute; + top: 100%; + left: 50%; + margin-left: -5px; + border-width: 5px; + border-style: solid; + border-color: #555 transparent transparent transparent; +} + +/* Toggle this class - hide and show the popup */ +.popupD4S .show { + visibility: visible; + -webkit-animation: fadeIn 1s; + animation: fadeIn 1s; +} + +/* Add animation (fade in the popup) */ +@-webkit-keyframes fadeIn { + from {opacity: 0;} + to {opacity: 1;} +} + +@keyframes fadeIn { + from {opacity: 0;} + to {opacity:1 ;} +} + +/*==================================== +D4S PACKAGE +======================================*/ + +.graphic-preview-style { + text-align: center; + border-top: 1px dotted #DDD; + padding-top: 10px; + padding-bottom: 0px; + margin-top: 15px; +} + +.graphic-preview-style a{ + font-size: 13px; +} + +.graphic-preview-style img{ + max-width: 100% !important; + height: auto; + +} + +.graphic-preview-style #graphic-title{ + font-size: 13px; + +} + +.nav-item{ + word-wrap:break-word; + } + +/*==================================== +RESOURCE_LIST RESOURCE_ITEM INTO PACKAGE +======================================*/ + +.required-access { + font-style: italic; + font-weight: bold; + padding: 5px; +} + +/*==================================== +LINK TO RESOURCES FROM PACKAGE LIST +======================================*/ + +.dataset-resources li a { + background-color: #187794; +} + +.label[data-format="csw"], .label[data-format*="csw"] { + background-color: #e6b800; +} + +/*==================================== +CSS APPLIED TO Similar GRSF Records +======================================*/ + +.my-grsf-table{ + word-break: break-all; +} + +.my-grsf-table tr td{ + width: inherit; +} + +.my-grsf-table tr td:first-child{ + width: 82px !important; +} + +/*==================================== +CSS APPLIED in base.html +======================================*/ + +#ckan-page-loading { + display: none; + position: fixed; + top: 50%; + left: 50%; + margin-top: -130px; + margin-left: -130px; + width: 260px; + height: 260px; + z-index: 100000; + background-image: url("/pageloading.gif"); + background-repeat: no-repeat; + background-position: center; +} + +/*==================================== +CSS APPLIED in search_for_location.html +======================================*/ + +div#search-for-location { + +} + +div#search-for-location #dataset-map { + position: relative !important; + top: +0px !important; +} + +div#search-for-location #dataset-map-container { + height: 300px; +} + + +div#search-for-location .module-heading { + display: none; +} + +div#search-for-extent{ + padding-top: 10px; +} + +/*==================================== +CSS APPLIED in additional_info.html +======================================*/ +.qr-code-table { + width: 100%; +} + +.qr-code-table td { + width: 85%; + border: 1px solid #e3e3e3; +} + +.qr-code-table td:first-child { + padding-left: 10px; + border-right-style: none; + +} + +.qr-code-table td:last-child { + width: 105px; + text-align: center; + border-left-style: none; + +} + +/* MAX-WITH APPIED TO QR_CODE */ +.qr-code-table img { + max-width: 100px; + height: auto; +} + + +/*==================================== +CSS APPLIED FROM JSON TO HTML TABLE +======================================*/ + +.json-to-html-table-column{ + word-break: break-all; +} + +.json-to-html-table-column tr td{ + width: inherit; +} + +.json-to-html-table-column tr td:first-child{ + font-weight: bold; + color: #5a5a5a; +} + +/*==================================== +CSS APPLIED into custom_form_fields +======================================*/ +.disabled-div{ + pointer-events: none; + opacity: 0.5; +} + +.disabled-div input[type="text"]{ + background: #f1f1f1; +} + +/*==================================== +CSS APPLIED into extra_table.html +======================================*/ + +.read-more-state { + display: none; +} + +.read-more-target { + opacity: 0; + max-height: 0; + font-size: 0; + transition: .25s ease; +} + +.read-more-state:checked ~ .read-more-wrap .read-more-target { + opacity: 1; + font-size: inherit; + max-height: 999em; + content: ""; +} + +.read-more-state ~ .read-more-trigger:before { + content: 'Show more'; +} + +.read-more-state:checked ~ .read-more-trigger:before { + content: 'Show less'; +} + +.read-more-state:checked ~ .read-more-wrap::after { + content: ""; +} + +.read-more-trigger { + cursor: pointer; + display: inline-block; + padding: 0 .5em; + color: #187794; + font-size: .9em; + line-height: 2; + border: 1px solid #ddd; + border-radius: .25em; + font-weight: normal; +} + +.read-more-trigger::after { + content: ""; +} + +.read-more-wrap { + margin-bottom: 2px; +} + +.read-more-wrap::after{ + content: " ..."; + +} \ No newline at end of file diff --git a/ckanext/d4science/helpers.py b/ckanext/d4science/helpers.py index 98b7f2d..157c5f4 100644 --- a/ckanext/d4science/helpers.py +++ b/ckanext/d4science/helpers.py @@ -1,9 +1,753 @@ -def d4science_hello(): - return "Hello, d4science!" +import ckan.authz as authz +import ckan.model as model +from webhelpers2.html import escape, HTML, literal +from webhelpers2.text import truncate +import ckan.lib.helpers as h +import ckan.logic as logic +from ckan.common import config +from ckanext.d4science.d4sdiscovery.d4s_namespaces_controller import D4S_Namespaces_Controller +from ckanext.d4science.d4sdiscovery.d4s_namespaces_extras_util import D4S_Namespaces_Extra_Util +from ckanext.d4science.qrcodelink.generate_qrcode import D4S_QrCode +import urllib.request, urllib.error + +from ckan.common import ( + _, ungettext, g, c, request, session, json +) + +from flask import Blueprint, render_template, g, request, url_for, current_app + +import random +import webhelpers2 +from operator import itemgetter +from logging import getLogger +import base64 +import sys, os, re +import configparser +import collections +from collections import OrderedDict + +log = getLogger(__name__) + +systemtype_field = 'systemtypefield' +systemtype_field_default_value = 'system:type' +ic_proxy_url_field = 'ic_proxy_url' +ic_proxy_url_field_default_value = "https://registry.d4science.org/icproxy/gcube/service" +application_token_field = 'application_token' +namespaces_generic_resource_id_default_value = "23d827cd-ba8e-4d8c-9ab4-6303bdb7d1db" +namespaces_gr_id_fieldname = "namespaces_generic_resource_id" +namespaceseparator_field = 'namespace_separator' +namespaceseparator_field_default_value = ':' +systemtype_rgb_colors = ['#c0392b ', '#585858', '#04407C', '#9b59b6', '#2ecc71', '#16a085', '#7f8c8d ', '#2ecc71', + '#FA8072', '#00FFFF', '#C76611', '#f39c12', '#800000'] +systemtype_field_colors = 'systemtype_field_colors' + +systemtype_cms_fields_placeholders = {'prefix': 'system:cm_', 'item_status': 'system:cm_item_status'} + +NOCATEOGORY = 'nocategory' +TRANSLATE_OF_ = 'translate_of_' + +ctg_namespace_ctrl = None -def get_helpers(): +def get_user_role_for_group_or_org(group_id, user_name): + ''' Returns the user's role for the group. (Ignores privileges that cascade + in a group hierarchy.)''' + return authz.users_role_for_group_or_org(group_id, user_name) + +def get_parents_for_group(group_name_or_id): + ''' Returns the user's role for the group. (Ignores privileges that cascade + in a group hierarchy.)''' + group = model.Group.get(group_name_or_id) + if group: + return model.Group.get_parent_group_hierarchy(group) + else: + return None + + +def get_header_param(parameter_name, default=None): + ''' This function allows templates to access header string parameters + from the request. ''' + return request.headers.get(parameter_name, default) + + +def get_request_param(parameter_name, default=None): + ''' This function allows templates to access query string parameters + from the request. ''' + return request.args.get(parameter_name, default) + #return request.params.get(parameter_name, default) + + +def get_cookie_value(cookie_name, default=None): + ''' This function allows templates to access cookie by cookie_name parameter + from the request. ''' + + value = request.cookies.get(cookie_name) + + if value is None: + print('cookie: ' + cookie_name + ', has value None') + else: + print('cookie: ' + cookie_name + ', has value ' + value) + + return value + + +def markdown_extract_html(text, extract_length=190, allow_html=False): + ''' Returns the plain text representation of markdown encoded text. That + is the texted without any html tags. If extract_length is 0 then it + will not be truncated.''' + if not text: + return '' + if allow_html: + plain = h.markdown(text.strip()) + else: + plain = h.RE_MD_HTML_TAGS.sub('', h.markdown(text)) + + if not extract_length or len(plain) < extract_length: + return literal(plain) + return literal(str(truncate(plain, length=extract_length, indicator='...', whole_word=True))) + + +def get_systemtype_field_dict_from_session(): + '''Return the value of 'ckan.d4science_theme.metadatatypefield' + read from production.ini''' + + systemtype_fieldname = session.get(systemtype_field) + + if systemtype_fieldname is None: + log.info(systemtype_field + " not found in session, loading from config") + else: + log.debug(systemtype_field + " found in session having value: %s" % systemtype_fieldname) + return systemtype_fieldname + + systemtype_fieldname = current_app.config.get('ckan.d4science_theme.' + systemtype_field) + #systemtype_fieldname = config.get('ckan.d4science_theme.' + systemtype_field) + + if systemtype_fieldname is None: + log.info( + systemtype_field + " field does not exist in production.ini, returning default value %s" % systemtype_field_default_value) + systemtype_fieldname = systemtype_field_default_value + + separator = get_namespace_separator_from_session() + log.debug("Replacing %s" % separator + " with empty string for key %s" % systemtype_field) + systemtype_fieldname_name = systemtype_fieldname.replace(separator, "") + purgedfieldname = purge_namespace_to_fieldname(systemtype_fieldname) + log.debug("Setting %s" % systemtype_fieldname + " in session for key %s" % systemtype_field) + session[systemtype_field] = {'id': systemtype_fieldname, 'name': systemtype_fieldname_name, + 'title': purgedfieldname} + session.modified = True + #session.save() old pylons(?) + return session[systemtype_field] + + +def get_d4s_namespace_controller(): + '''Instance the D4S_Namespaces_Controller and check that the namespaces are not empty reading it from IS and/or using a Caching system. + The ic-proxy-url is built by reading the configurations from production.ini''' + + d4s_extras_controller = D4S_Namespaces_Controller.getInstance() + global ctg_namespace_ctrl + + if ctg_namespace_ctrl is not None: + log.info("ctg_namespace_ctrl with configurations is NOT None") + the_namespaces = d4s_extras_controller.load_namespaces(ctg_namespace_ctrl['ic_proxy_url'], + ctg_namespace_ctrl['resource_id'], + ctg_namespace_ctrl['application_token']) + log.debug("the_namespaces are %s" % the_namespaces) + + if the_namespaces is None or len(the_namespaces) == 0: + log.info("D4S_Namespaces_Controller obj with none or empty namespaces, going to read them") + else: + log.info("d4s_namespaces_controller found and the namespaces property is not empty: %s" % d4s_extras_controller) + return d4s_extras_controller + else: + log.info("ctg_namespace_ctrl with configurations is None, instancing it") + + ic_proxy_url_value = current_app.config.get('ckan.d4science_theme.' + ic_proxy_url_field) + #ic_proxy_url_value = config.get('ckan.d4science_theme.' + ic_proxy_url_field) old + + if ic_proxy_url_value is None: + log.info( + "ckan.d4science_theme." + ic_proxy_url_field + " field does not exist in production.ini, returning default value %s" % ic_proxy_url_field_default_value) + ic_proxy_url_value = ic_proxy_url_field_default_value + + application_token_fieldname = current_app.config.get('ckan.d4science_theme.' + application_token_field) + #application_token_fieldname = config.get('ckan.d4science_theme.' + application_token_field) + + if application_token_fieldname is None: + log.error("ckan.d4science_theme." + application_token_field + " field does not exist in production.ini!!!") + application_token_fieldname = None + + namespaces_gr_id_fieldname_value = current_app.config.get('ckan.d4science_theme.' + namespaces_gr_id_fieldname) + #namespaces_gr_id_fieldname_value = config.get('ckan.d4science_theme.' + namespaces_gr_id_fieldname) + + if namespaces_gr_id_fieldname_value is None: + log.error("ckan.d4science_theme." + application_token_field + " field does not exist in production.ini!!!") + namespaces_gr_id_fieldname_value = namespaces_generic_resource_id_default_value + + # filling the ctg_namespace_ctrl with IS configurations to perform the query for loading the namespaces from IS + ctg_namespace_ctrl = {'ic_proxy_url': ic_proxy_url_value, + 'application_token': application_token_fieldname, + 'resource_id': namespaces_gr_id_fieldname_value} + + d4s_extras_controller.load_namespaces(ctg_namespace_ctrl['ic_proxy_url'], ctg_namespace_ctrl['resource_id'], + ctg_namespace_ctrl['application_token']) + + return d4s_extras_controller + + +def get_extras_indexed_for_namespaces(extras): + namespace_dict = get_namespaces_dict() + # log.info("my_namespace_dict %s" % namespace_dict) + my_extra = get_extras(extras) + # log.info("my_extra is %s" % my_extra) + # d4s_extras_controller = D4S_Namespaces_Controller.getInstance() + # extras_indexed_for_categories = d4s_extras_controller.get_extras_indexed_for_namespaces(namespace_dict, my_extra) + + extras_indexed_for_categories = D4S_Namespaces_Extra_Util().get_extras_indexed_for_namespaces(namespace_dict, + my_extra) + return extras_indexed_for_categories + + +def get_namespaces_dict(): + d4s_extras_controller = get_d4s_namespace_controller() + + if d4s_extras_controller is not None: + return d4s_extras_controller.get_dict_ctg_namespaces() + else: + log.info("local_extras_controller is null, returning empty dictionary for namespaces") + return {} + + +def get_extra_for_category(extras_indexed_for_categories, key_category): + if key_category in extras_indexed_for_categories: + catalogue_namespace = extras_indexed_for_categories[key_category] + return catalogue_namespace.extras + + return [] + + +def get_systemtype_value_from_extras(package, extras=None): + '''Returns the value of metadata fied read from key 'metadatatype' + stored into extra fields if it exists, 'No Type' otherwise''' + systemtype_dict = get_systemtype_field_dict_from_session() + + no_type = 'No Type' + + if extras is None: + return no_type + + for extra in extras: + k, v = extra['key'], extra['value'] + log.debug("key is %s" % k) + log.debug("value is %s" % v) + if k == str(systemtype_dict['id']): + return v + + return no_type + + +def get_namespace_separator_from_session(): + '''Returns the character used to separate namespace from fieldname''' + + separator = session.get(namespaceseparator_field) + + if separator is None: + log.info(namespaceseparator_field + " not found in session, loading from config") + else: + log.debug(namespaceseparator_field + " found in session: %s" % separator) + return separator + + namespace_sep = config.get('ckan.d4science_theme.' + namespaceseparator_field) + + if namespace_sep is None: + log.info( + namespaceseparator_field + " field does not exist in production.ini, returning default value %s" % namespaceseparator_field_default_value) + namespace_sep = namespaceseparator_field_default_value + + log.debug("Setting %s" % namespace_sep + " in session for key %s" % namespaceseparator_field) + session[namespaceseparator_field] = namespace_sep + return namespace_sep + + +def get_extras(package_extras, auto_clean=False, subs=None, exclude=None): + ''' Used for outputting package extras + + :param package_extras: the package extras + :type package_extras: dict + :param auto_clean: If true capitalize and replace -_ with spaces + :type auto_clean: bool + :param subs: substitutes to use instead of given keys + :type subs: dict {'key': 'replacement'} + :param exclude: keys to exclude + :type exclude: list of strings + ''' + + # If exclude is not supplied use values defined in the config + if not exclude: + exclude = g.package_hide_extras + output = [] + for extra in package_extras: + if extra.get('state') == 'deleted': + continue + k, v = extra['key'], extra['value'] + if k in exclude: + continue + if subs and k in subs: + k = subs[k] + elif auto_clean: + k = k.replace('_', ' ').replace('-', ' ').title() + if isinstance(v, (list, tuple)): + v = ", ".join(map(str, v)) + output.append((k, v)) + return output + + +def purge_namespace_to_fieldname(fieldname): + separator = get_namespace_separator_from_session() + + if fieldname is None: + return "" + + if separator not in fieldname: + return fieldname + + end = fieldname.index(separator) + 1 + max_l = len(fieldname) + if end < max_l: + return fieldname[end:max_l] + return fieldname + + +def purge_namespace_to_string(facet): + if not g.search_facets or \ + not g.search_facets.get(facet) or \ + not g.search_facets.get(facet).get('items'): + return "" + + facet_name = g.search_facets.get(facet) + end = str(facet_name).index(":") + if end <= len(facet_name): + return facet_name[:end] + return facet_name + + +def count_facet_items_dict(facet, limit=None, exclude_active=False): + if not g.search_facets or \ + not g.search_facets.get(facet) or \ + not g.search_facets.get(facet).get('items'): + return 0 + facets = [] + for facet_item in g.search_facets.get(facet)['items']: + if not len(facet_item['name'].strip()): + continue + if not (facet, facet_item['name']) in list(request.args.items()): + facets.append(dict(active=False, **facet_item)) + elif not exclude_active: + facets.append(dict(active=True, **facet_item)) + + # for count, + total = len(facets) + log.debug("total facet: %s are %d" % (facet, total)) + return total + + +def random_color(): + rgbl = [255, 0, 0] + random.shuffle(rgbl) + return tuple(rgbl) + + +def check_url(the_url): + try: + urllib.request.urlopen(the_url) + return True + except urllib.error.HTTPError as e: + return False + except urllib.error.URLError as e: + return False + except Exception as error: + return False + + +def get_color_for_type(systemtype_field_value): + '''Return a color assigned to a system type''' + + systemtypecolors = session.get(systemtype_field_colors) + + if systemtypecolors is None: + log.info("color: " + systemtype_field_colors + " not found in session, creating new one") + systemtypecolors = {} + session[systemtype_field_colors] = systemtypecolors + else: + log.debug("color: " + systemtype_field_colors + " found in session having value: %s" % systemtypecolors) + + e_color = systemtypecolors.get(systemtype_field_value) + + if e_color is None: + usedcolorsLen = len(systemtypecolors) + colorsLen = len(systemtype_rgb_colors) + index = usedcolorsLen if usedcolorsLen < colorsLen else random.randint(0, colorsLen - 1) + e_color = systemtype_rgb_colors[index] + # log.debug("color: adding color %s" %e_color +" index is: "+str(index)) + systemtypecolors[systemtype_field_value] = e_color + session[systemtype_field_colors] = systemtypecolors + + session.modified = True + #session.save() + # log.debug("color: returning color %s" %e_color +" for type: "+systemtype_field_value) + return e_color + + +def ordered_dictionary(list_to_be_sorted, property='name', ordering="asc"): + + ord = False if ordering == "asc" else True + + if list_to_be_sorted: + return sorted(list_to_be_sorted, key=itemgetter(property), reverse=ord) + + return list_to_be_sorted + + +def qrcode_for_url(url): + if url: + try: + qr_code = D4S_QrCode(url) + image_path = qr_code.get_qrcode_path() + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()) + return "" + except Exception as error: + log.error("Error on getting qrcode for url: " + url + "error: %s" % error) + + return "" + + +def get_list_of_organizations(limit=10, sort='packages'): + to_browse_organizations = [] + try: + data = {} + + if sort: + data['sort'] = sort + + data['limit'] = limit + data['all_fields'] = True + ordered_organizations = [] + ordered_organizations = logic.get_action('organization_list')({}, data) + + for organization in ordered_organizations: + try: + to_browse_obj = {} + + if not organization['name']: + continue + + to_browse_obj['name'] = organization['name'] + + if 'package_count' in organization: + to_browse_obj['package_count'] = organization['package_count'] + + if 'display_name' in organization: + to_browse_obj['display_name'] = organization['display_name'] + + image_url = get_url_to_icon_for_ckan_entity(organization['name'], 'organization', False) + + # Using ICON as first option + if image_url: + to_browse_obj['url'] = image_url + # Using object image_url as second one + elif 'image_url' in organization and organization['image_url']: + to_browse_obj['url'] = organization['image_url'] + # Default placeholder + else: + to_browse_obj['url'] = h.url_for('static', filename='images/organisations/icon/placeholder-organization.png') + + to_browse_organizations.append(to_browse_obj) + except (logic.NotFound, logic.ValidationError, logic.NotAuthorized) as error: + log.warning("Error on putting organization: %s" % error) + + log.info("browse %d" % len(ordered_organizations) + " organisation/s") + except (logic.NotFound, logic.ValidationError, logic.NotAuthorized) as error: + log.error("Error on getting organizations: %s" % error) + return [] + + return to_browse_organizations + + +def get_list_of_groups(limit=10, sort='package_count'): + to_browse_groups = [] + try: + data = {} + if sort: + data['sort'] = sort + + data['limit'] = limit + data['all_fields'] = True + ordered_groups = [] + ordered_groups = logic.get_action('group_list')({}, data) + + for group in ordered_groups: + try: + to_browse_obj = {} + + if not group['name']: + continue + + to_browse_obj['name'] = group['name'] + + if 'package_count' in group: + to_browse_obj['package_count'] = group['package_count'] + + if 'display_name' in group: + to_browse_obj['display_name'] = group['display_name'] + + if 'image_url' in group and group['image_url']: + to_browse_obj['url'] = group['image_url'] + else: + to_browse_obj['url'] = get_url_to_icon_for_ckan_entity(group['name'], 'group') + + to_browse_groups.append(to_browse_obj) + except (logic.NotFound, logic.ValidationError, logic.NotAuthorized) as error: + log.warning("Error on putting group: %s" % error) + + log.info("browse %d" % len(ordered_groups) + " organisation/s") + except (logic.NotFound, logic.ValidationError, logic.NotAuthorized) as error: + log.error("Error on getting group: %s" % error) + return [] + + return to_browse_groups + + +def get_browse_info_for_organisations_or_groups(type='organization', limit=10, sort_field=None): + sort = None + if sort_field: + sort = sort_field + + if type == 'organization': + if sort: + return get_list_of_organizations(limit, sort) + else: + return get_list_of_organizations(limit) + + elif type == 'group': + if sort: + return get_list_of_groups(limit, sort) + else: + return get_list_of_groups(limit) + + return [] + + +def get_image_display_for_group(item_id): + if item_id: + try: + item_obj = model.Group.get(item_id) + + if item_obj and item_obj.image_url: + return item_obj.image_url + else: + return h.url_for('static', filename='images/groups/icon/placeholder-group.png') + + except Exception as error: + log.error("Error on getting item obj: %s" % item_id + "error: %s" % error) + + +def get_application_path(): + if getattr(sys, 'frozen', False): + # If the application is run as a bundle, the pyInstaller bootloader + # extends the sys module by a flag frozen=True and sets the app + # path into variable _MEIPASS'. + application_path = sys._MEIPASS + else: + application_path = os.path.dirname(os.path.abspath(__file__)) + + return application_path + + +''' +Get icon url for input entity type +@:param default_placeholder if True returns the URL of default image, otherwise None. +''' + + +def get_url_to_icon_for_ckan_entity(item_name, entity_type=None, default_placeholder=True): + if not entity_type or not item_name: + return None + + dir_images_full_path = get_application_path() + "/public/images" + dir_images_relative_path = "/images" + + if entity_type == 'group': + dir_images_full_path += "/groups" + dir_images_relative_path += "/groups" + placeholder_icon = "placeholder-group.png" + elif entity_type == 'organization': + dir_images_full_path += "/organisations" + dir_images_relative_path += "/organisations" + placeholder_icon = "placeholder-organization.png" + elif entity_type == 'type': + dir_images_full_path += "/types" + dir_images_relative_path += "/types" + placeholder_icon = "placeholder-type.png" + else: + return None + + icon_path = os.path.join(dir_images_full_path, "icon", item_name.lower() + ".png") + if os.path.isfile(icon_path): + return h.url_for('static', filename=os.path.join(dir_images_relative_path, "icon", item_name.lower() + ".png")) + elif default_placeholder: + return h.url_for('static', filename=os.path.join(dir_images_relative_path, "icon", placeholder_icon)) + + return None + + +def get_user_info(user_id_or_name): + if user_id_or_name: + try: + + item_obj = model.User.get(user_id_or_name) + + if item_obj: + return item_obj + + return None + except Exception as error: + log.error("Error on getting item obj: %s" % user_id_or_name + "error: %s" % error) + + return None + + +''' +Search the value of my_search_string into input file {ckan_po_file} or the default file ckan.po provided as CKAN language +and returns its translate +''' + + +def get_ckan_translate_for(ckan_po_file, my_search_string): + my_translate = session.get(TRANSLATE_OF_ + my_search_string) + + if not my_search_string: + return "" + + if my_translate: + log.info("Translate of '%s' " % my_search_string + " found in session as: %s" % my_translate) + return my_translate + + if not ckan_po_file: + ckan_po_file = "/usr/lib/ckan/default/src/ckan/ckan/i18n/en_gcube/LC_MESSAGES/ckan.po" + + numlines = 0 + numfound = 0 + found = 0 + line_text = "" + + try: + infile = open(ckan_po_file, "r") + + for line in infile: + numlines += 1 + if found > 0: + numfound += 1 + line_text += str(line) + found = 0 # reset found + + # found += line.upper().count(my_search_string.upper()) + found += line.count(my_search_string) + + if found > 0: + log.debug("The search string '%s'" % my_search_string + " was found. Read the line: %s" % str(line)) + + infile.close() + + except Exception as e: + print("Exception during parsing the file %s" % ckan_po_file, e) + + log.info("Recap: '%s' was found" % my_search_string + " %i times " % numfound + "in %i lines" % numlines) + log.debug("Line text is: %s" % line_text) + + pattern = '"([A-Za-z0-9_ \./\\-]*)"' + m = re.search(pattern, line_text) + + try: + my_translate = m.group() + except Exception as e: + print("Pattern %s" % my_search_string + " not found ", e) + + if my_translate: + log.debug("Replacing quotas...") + my_translate = my_translate.replace("\"", "") + + log.info("Found the string '%s'" % my_translate + " that translating '%s'" % my_search_string) + + session[TRANSLATE_OF_ + my_search_string] = my_translate + session.modified = True + #session.save() capire se serve, approccio standard con modified, save forza il salvataggio esplicito della sessione + + return my_translate + + +def get_location_to_bboxes(): + config = configparser.ConfigParser() + config.optionxform = str + location_to_bboxes = {} + try: + bboxes_file = get_application_path() + "/public/location_to_bboxes.ini" + log.debug("bboxes_file is: '%s'" % bboxes_file) + config.read(bboxes_file) + for section_name in config.sections(): + log.debug('Location to bboxes Section: ' + section_name) + # print ' Options:', parser.options(section_name) + for name, value in config.items(section_name): + location_to_bboxes[name] = value.replace(",", "%2C") + + ordDictBboxes = collections.OrderedDict(sorted(location_to_bboxes.items())) + log.debug("Ordered 'bboxes_file' dict: '%s'" % ordDictBboxes) + return ordDictBboxes + except Exception as error: + log.error("Error on reading file: %s" % bboxes_file + "error: %s" % error) + +def get_content_moderator_system_placeholder(): + return systemtype_cms_fields_placeholders + + +#ITemplateHelpers +def get_helpers(self): + log.info("get_helpers called...") + '''Register functions as a template + helper function. + ''' + # Template helper function names should begin with the name of the + # extension they belong to, to avoid clashing with functions from + # other extensions. return { - "d4science_hello": d4science_hello, + 'd4science_theme_get_user_role_for_group_or_org': get_user_role_for_group_or_org, + 'd4science_theme_get_parents_for_group': get_parents_for_group, + 'get_header_param': get_header_param, + 'get_request_param': get_request_param, + 'get_cookie_value': get_cookie_value, + 'd4science_theme_markdown_extract_html' : markdown_extract_html, + 'd4science_theme_get_systemtype_value_from_extras' : get_systemtype_value_from_extras, + 'd4science_theme_get_systemtype_field_dict_from_session' : get_systemtype_field_dict_from_session, + 'd4science_theme_get_namespace_separator_from_session' : get_namespace_separator_from_session, + 'd4science_theme_get_extras' : get_extras, + 'd4science_theme_count_facet_items_dict' : count_facet_items_dict, + 'd4science_theme_purge_namespace_to_facet': purge_namespace_to_fieldname, + 'd4science_get_color_for_type': get_color_for_type, + 'd4science_get_d4s_namespace_controller': get_d4s_namespace_controller, + 'd4science_get_extras_indexed_for_namespaces': get_extras_indexed_for_namespaces, + 'd4science_get_namespaces_dict': get_namespaces_dict, + 'd4science_get_extra_for_category' : get_extra_for_category, + 'd4science_get_ordered_dictionary': ordered_dictionary, + 'd4science_get_qrcode_for_url': qrcode_for_url, + 'd4science_get_list_of_organizations': get_list_of_organizations, + 'd4science_get_image_display_for_group': get_image_display_for_group, + 'd4science_get_list_of_groups': get_list_of_groups, + 'd4science_get_browse_info_for_organisations_or_groups': get_browse_info_for_organisations_or_groups, + 'd4science_get_user_info': get_user_info, + 'd4science_get_url_to_icon_for_ckan_entity' : get_url_to_icon_for_ckan_entity, + 'd4science_get_ckan_translate_for' : get_ckan_translate_for, + 'd4science_get_location_to_bboxes' : get_location_to_bboxes, + 'd4science_get_content_moderator_system_placeholder': get_content_moderator_system_placeholder, } + + diff --git a/ckanext/d4science/plugin.py b/ckanext/d4science/plugin.py index 1d886d8..b07ddcf 100644 --- a/ckanext/d4science/plugin.py +++ b/ckanext/d4science/plugin.py @@ -1,15 +1,206 @@ import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit +import ckan.lib.dictization.model_save as model_save +import sqlalchemy as sa from ckan.views.resource import Blueprint from ckanext.d4science import helpers from ckanext.d4science.logic import action, auth from ckanext.d4science.logic import validators +from ckan.config.middleware.common_middleware import TrackingMiddleware from ckan.model import Package +from logging import getLogger +from flask import g + +log = getLogger(__name__) + +def remove_check_replicated_custom_key(schema): + if schema is not None: + schema.pop('__before', None) + + return schema + +def _package_extras_save(extra_dicts, obj, context): + ''' It can save repeated extras as key-value ''' + #allow_partial_update = context.get("allow_partial_update", False) potrebbe non servire + if extra_dicts is None: #and allow_partial_update: + return + + model = context["model"] + session = context["session"] + + log.debug("extra_dicts: "+ str(extra_dicts)) + #print "extra_dicts: "+str(extra_dicts) + + extras_list = obj.extras_list + #extras = dict((extra.key, extra) for extra in extras_list) + old_extras = {} + extras = {} + for extra in extras_list or []: + old_extras.setdefault(extra.key, []).append(extra.value) + extras.setdefault(extra.key, []).append(extra) + + #print "old_extras: "+str(old_extras) + + new_extras = {} + for extra_dict in extra_dicts or []: + #print 'extra_dict key: '+extra_dict["key"] + ', value: '+extra_dict["value"] + #new_extras.setdefault(extra_dict["key"], []).append(extra_dict["value"]) + if extra_dict.get("deleted"): + log.debug("extra_dict deleted: "+str(extra_dict["key"])) + #print 'extra_dict deleted: '+extra_dict["key"] + continue + + #if extra_dict['value'] is not None and not extra_dict["value"] == "": + if extra_dict['value'] is not None: + new_extras.setdefault(extra_dict["key"], []).append(extra_dict["value"]) + + log.debug("new_extras: "+ str(new_extras)) + #print "new_extras: "+str(new_extras) + + #aggiunta di nuove chiavi + for key in set(new_extras.keys()) - set(old_extras.keys()): + #state = 'active' + log.debug("adding key: " + str(key)) + #print "adding key: "+str(key) + extra_lst = new_extras[key] + for extra in extra_lst: + extra = model.PackageExtra(state='active', key=key, value=extra) + session.add(extra) + extras_list.append(extra) + + #gestione chiavi eliminate + for key in set(old_extras.keys()) - set(new_extras.keys()): + log.debug("deleting key: "+ str(key)) + #print "deleting key: "+str(key) + extra_lst = extras[key] + for extra in extra_lst: + #state = 'deleted' + extra.state = 'deleted' + extras_list.remove(extra) + + #gestione chiavi aggiornate + for key in set(new_extras.keys()) & set(old_extras.keys()): + #for each value of new list + for value in new_extras[key]: + old_occur = old_extras[key].count(value) + new_occur = new_extras[key].count(value) + log.debug("value: " + str(value) + ", new_occur: "+ str(new_occur)+ ", old_occur: "+ str(old_occur)) + #print "value: "+str(value) + ", new_occur: "+str(new_occur) + ", old_occur: "+str(old_occur) + # it is an old value deleted or not + if value in old_extras[key]: + if old_occur == new_occur: + #print "extra - occurrences of: "+str(value) +", are equal into both list" + log.debug("extra - occurrences of: "+ str(value) +", are equal into both list") + #there is a little bug, this code return always the first element, so I'm fixing with #FIX-STATUS + extra_values = get_package_for_value(extras[key], value) + #extras_list.append(extra) + for extra in extra_values: + #state = 'active' + extra.state = 'active' + session.add(extra) + #print "extra updated: "+str(extra) + log.debug("extra updated: "+ str(extra)) + + elif new_occur > old_occur: + #print "extra - a new occurrence of: "+str(value) +", is present into new list, adding it to old list" + log.debug("extra - a new occurrence of: "+ str(value) + ", is present into new list, adding it to old list") + #state = 'active' + extra = model.PackageExtra(state='active', key=key, value=value) + #extra.state = state + #extra.state = 'active' non dovrebbe servire + session.add(extra) + extras_list.append(extra) + old_extras[key].append(value) + log.debug("old extra values updated: "+ str(old_extras[key])) + #print "old extra values updated: "+str(old_extras[key]) + + else: + #remove all occurrences deleted - this code could be optimized, it is run several times but could be performed one shot + countDelete = old_occur-new_occur + log.debug("extra - occurrence of: "+ str(value).encode('utf-8') + ", is not present into new list, removing "+ str(countDelete).encode('utf-8')+" occurrence/s from old list") + #print "extra - occurrence of: "+str(value) +", is not present into new list, removing "+str(countDelete)+" occurrence/s from old list" + extra_values = get_package_for_value(extras[key], value) + for idx, extra in enumerate(extra_values): + if idx < countDelete: + #print "extra - occurrence of: "+str(value) +", is not present into new list, removing it from old list" + log.debug("pkg extra deleting: "+ str(extra.value)) + #print "pkg extra deleting: "+str(extra.value) + #state = 'deleted' + extra.state = 'deleted' + + else: + #print "pkg extra reactivating: "+str(extra.value) + log.debug("pkg extra reactivating: "+ str(extra.value)) + #state = 'active' + extra.state = 'active' + session.add(extra) + + else: + #print "extra new value: "+str(value) + log.debug("extra new value: " + str(value)) + #state = 'active' + extra = model.PackageExtra(state='active', key=key, value=value) + #extra.state = state + #extra.state = 'active' + session.add(extra) + extras_list.append(extra) + + + #chiavi vecchie non presenti + for value in old_extras[key]: + #if value is not present in new list + if value not in new_extras[key]: + extra_values = get_package_for_value(extras[key], value) + for extra in extra_values: + #print "not present extra deleting: "+str(extra) + log.debug("not present extra deleting: "+ str(extra)) + #state = 'deleted' + extra.state = 'deleted' + +def get_package_for_value(list_package, value): + ''' Returns a list of packages containing the value passed in input''' + + return [x for x in list_package if x.value == value] + #lst = [] + #for x in list_package: + # if x.value == value: + # lst.append(x) + # else: + # return lst + # + #return lst + + +#OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE +#gestisce le connessioni al db relazionale utilizzato da ckan +def _init_TrackingMiddleware(self, app, config): + self.app = app + log.debug('TrackingMiddleware d4Science instance') + sqlalchemy_url = config.get('sqlalchemy.url') + log.debug('sqlalchemy_url read: '+str(sqlalchemy_url)) + + sqlalchemy_pool = config.get('sqlalchemy.pool_size') + if sqlalchemy_pool is None: + sqlalchemy_pool = 5 + + log.debug('sqlalchemy_pool read: '+str(sqlalchemy_pool)) + sqlalchemy_overflow = config.get('sqlalchemy.max_overflow') + + if sqlalchemy_overflow is None: + sqlalchemy_overflow = 10 + + log.debug('sqlalchemy_overflow read: '+str(sqlalchemy_overflow)) + + try: + self.engine = sa.create_engine(sqlalchemy_url, pool_size=int(sqlalchemy_pool), max_overflow=int(sqlalchemy_overflow)) + except TypeError as e: + log.error('pool size does not work: ' +str(e.args)) + self.engine = sa.create_engine(sqlalchemy_url) # import ckanext.d4science.cli as cli # import ckanext.d4science.helpers as helpers -# import ckanext.d4science.views as views +import ckanext.d4science.views as views # from ckanext.d4science.logic import ( # action, auth, validators # ) @@ -51,47 +242,52 @@ class D4SciencePlugin(plugins.SingletonPlugin): def package_types(self): # Aggiunta del tipo di dato personalizzato deliverable - return ['deliverable_type'] + return [] def is_fallback(self): # Indica che questo plugin può essere usato come fallback se un tipo specifico non è specificato return False + + #IDatasetForm + def package_types(self): + # This plugin doesn't handle any special package types, it just + # registers itself as the default (above). + return [] def create_package_schema(self): schema = super(D4SciencePlugin, self).create_package_schema() - schema.update({ - 'deliverable_type': [ignore_missing, unicode], - }) + schema = remove_check_replicated_custom_key(schema) + #schema.update({ + # 'deliverable_type': [ignore_missing, unicode], + #}) return schema def update_package_schema(self): schema = super(D4SciencePlugin, self).update_package_schema() - schema.update({ - 'deliverable_type': [ignore_missing, unicode], - }) + schema = remove_check_replicated_custom_key(schema) + #schema.update({ + # 'deliverable_type': [ignore_missing, unicode], + #}) return schema def show_package_schema(self): schema = super(D4SciencePlugin, self).show_package_schema() - schema.update({ - 'deliverable_type': [ignore_missing, unicode], - }) + schema = remove_check_replicated_custom_key(schema) + #schema.update({ + # 'deliverable_type': [ignore_missing, unicode], + #}) return schema + #override + model_save.package_extras_save = _package_extras_save + + #OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE + TrackingMiddleware.__init__ = _init_TrackingMiddleware + # IBlueprint - # def get_blueprint(self): - # return views.get_blueprints() - def get_blueprint(self): - blueprint = Blueprint('foo', self.__module__) - # rules = [ - # ('/group', 'group_index', custom_group_index), - # ] - # for rule in rules: - # blueprint.add_url_rule(*rule) - - return blueprint + return views.get_blueprints() # IClick diff --git a/ckanext/d4science/public/batman_logo.png b/ckanext/d4science/public/batman_logo.png new file mode 100644 index 0000000..e1175ac Binary files /dev/null and b/ckanext/d4science/public/batman_logo.png differ diff --git a/ckanext/d4science/qrcodelink/__init__.py b/ckanext/d4science/qrcodelink/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ckanext/d4science/qrcodelink/generate_qrcode.py b/ckanext/d4science/qrcodelink/generate_qrcode.py new file mode 100644 index 0000000..24cf3a3 --- /dev/null +++ b/ckanext/d4science/qrcodelink/generate_qrcode.py @@ -0,0 +1,76 @@ +import os +import tempfile +import logging +import pyqrcode +import time + +CATALINA_HOME = 'CATALINA_HOME' +log = logging.getLogger(__name__) + +temp_dir = None +qr_code_dir = None +qr_code_dir_name = "qr_code_for_catalogue" + + +class D4S_QrCode(): + def __init__(self, qr_code_url=None): + self._qr_code_url = qr_code_url + global temp_dir + if temp_dir is None: + D4S_QrCode.init_temp_dir() + if temp_dir is None: + raise Exception('No temp directory found!') + + def get_qrcode_path(self): + image_name = self._qr_code_url.rsplit('/', 1)[-1] + image_name+=".svg" + image_path = os.path.join(qr_code_dir, image_name) + # ONLY IF QRCODE DOES NOT EXIST THEN IT WILL BE CREATED + if not os.path.isfile(image_path): + url = pyqrcode.create(self._qr_code_url) + url.svg(image_path, scale=3) + log.debug("Created QRCode image: " + image_name) + + attempt = 0 + while not os.path.exists(image_path) and attempt < 3: + time.sleep(1) + attempt += 1 + + if os.path.isfile(image_path): + log.info("QRcode image exists at: " + image_path) + else: + log.error("%s isn't a file!" % image_path) + + return image_path + + @classmethod + def init_temp_dir(cls): + global temp_dir + global qr_code_dir_name + global qr_code_dir + try: + temp_dir = str(os.environ[CATALINA_HOME]) + temp_dir = os.path.join(temp_dir, "temp") + except KeyError as error: + log.error("No environment variable for: %s" % CATALINA_HOME) + + if temp_dir is None: + temp_dir = tempfile.gettempdir() # using system tmp dir + + log.debug("Temp dir is: %s" % temp_dir) + + qr_code_dir = os.path.join(temp_dir, qr_code_dir_name) + + if not os.path.exists(qr_code_dir): + os.makedirs(qr_code_dir) + + def get_temp_directory(self): + return temp_dir + + def get_qr_code_dir(self): + return qr_code_dir + + +# D4S_QrCode.init_temp_dir() +#qr_code = D4S_QrCode("http://data.d4science.org/ctlg/BiodiversityLab/distribution_of_the_giant_squid_architeuthis") +#print qr_code.get_qrcode_path() diff --git a/ckanext/d4science/templates/footer.html b/ckanext/d4science/templates/footer.html new file mode 100644 index 0000000..2a91e5a --- /dev/null +++ b/ckanext/d4science/templates/footer.html @@ -0,0 +1,18 @@ +{% ckan_extends %} + +{% block d4science_footer_links %} + + {{ super() }} +

Footer test!

+ test + +{% endblock %} + +{% block footer_links_ckan %} + + {{ super() }} + +
  • Some Link
  • +
  • Another Link
  • + +{% endblock %} \ No newline at end of file diff --git a/ckanext/d4science/views.py b/ckanext/d4science/views.py index 7192356..ce17a6a 100644 --- a/ckanext/d4science/views.py +++ b/ckanext/d4science/views.py @@ -1,5 +1,9 @@ from flask import Blueprint +from ckanext.d4science.views_routes.home import d4science_home +from ckanext.d4science.views_routes.organization import organization_vre +from ckanext.d4science.views_routes.systemtype import d4s_type_blueprint + d4science = Blueprint( "d4science", __name__) @@ -14,4 +18,5 @@ d4science.add_url_rule( def get_blueprints(): - return [d4science] + all_blueprints = [d4science, d4science_home, organization_vre, d4s_type_blueprint] + return all_blueprints diff --git a/ckanext/d4science/views_routes/__init__.py b/ckanext/d4science/views_routes/__init__.py new file mode 100644 index 0000000..68f2e01 --- /dev/null +++ b/ckanext/d4science/views_routes/__init__.py @@ -0,0 +1,4 @@ +#The __init__.py files are required to make Python +#treat the directories as containing packages; this is done to prevent directories with a common name, such as string, from unintentionally hiding valid modules that occur later on the module search path. +#See: https://docs.python.org/3/tutorial/modules.html#packages + diff --git a/ckanext/d4science/views_routes/home.py b/ckanext/d4science/views_routes/home.py new file mode 100644 index 0000000..cc46afd --- /dev/null +++ b/ckanext/d4science/views_routes/home.py @@ -0,0 +1,72 @@ +#import logging +#from ckan.controllers.home import HomeController +from flask import Blueprint, render_template, g +import ckan.plugins as p +from ckan.common import OrderedDict, _, g +import ckan.lib.search as search +import ckan.model as model +import ckan.logic as logic +import ckan.lib.helpers as h + +#blueprint definition +d4science_home = Blueprint("d4science_home", __name__) + +#@d4science_home.route("/catalog") +@d4science_home.route("/") +def index(): + try: + # package search + context = {'model': model, 'session': model.Session,'user': g.user, 'auth_user_obj': g.userobj} + + facets = OrderedDict() + + default_facet_titles = { + 'organization': _('Organizations'), + 'groups': _('Groups'), + 'tags': _('Tags'), + 'res_format': _('Formats'), + 'license_id': _('Licenses'), + } + + for facet in g.facets: + if facet in default_facet_titles: + facets[facet] = default_facet_titles[facet] + else: + facets[facet] = facet + + # gestion filtri/facets + for plugin in p.PluginImplementations(p.IFacets): + facets = plugin.dataset_facets(facets, 'dataset') + + g.facet_titles = facets + + data_dict = { + 'q': '*:*', + 'facet.field': list(facets.keys()), + 'rows': 4, + 'start': 0, + 'sort': 'views_recent desc', + 'fq': 'capacity:"public"' + } + query = logic.get_action('package_search')(context, data_dict) + g.search_facets = query['search_facets'] + g.package_count = query['count'] + g.datasets = query['results'] + + #print "c.search_facets: " + #print " ".join(c.search_facets) + + except search.SearchError: + g.package_count = 0 + + if g.userobj and not g.userobj.email: + url = h.url_for('user.edit') + msg = _('Please update your profile' + ' and add your email address. ') % url + \ + _('%s uses your email address' + ' if you need to reset your password.') \ + % g.site_title + h.flash_notice(msg, allow_html=True) + + return render_template('home/index.html', cache_force=True) + diff --git a/ckanext/d4science/views_routes/organization.py b/ckanext/d4science/views_routes/organization.py new file mode 100644 index 0000000..23f7837 --- /dev/null +++ b/ckanext/d4science/views_routes/organization.py @@ -0,0 +1,118 @@ +# encoding: utf-8 +import re +import logging +from flask import Blueprint, g, request, abort, render_template +import ckan.plugins as plugins +import ckan.logic as logic +import ckan.model as model +import ckan.lib.helpers as h +import ckan.lib.search as search +from ckan.common import OrderedDict, _, NotAuthorized, NotFound + +organization_vre = Blueprint("organization_vre", __name__) + +''' The organization controller is for Organizations, which are implemented +as Groups with is_organization=True and group_type='organization'. It works +the same as the group controller apart from: +* templates and logic action/auth functions are sometimes customized + (switched using _replace_group_org) +* 'bulk_process' action only works for organizations + +Nearly all the code for both is in the GroupController (for historical +reasons). +''' + +group_types = ['organization'] + +def _guess_group_type(expecting_name=False): + return 'organization' + +def _replace_group_org( string): + ''' substitute organization for group if this is an org''' + return re.sub('^group', 'organization', string) + +def _update_facet_titles(facets, group_type): + for plugin in plugins.PluginImplementations(plugins.IFacets): + facets = plugin.organization_facets( + facets, group_type, None) + +@organization_vre.route('/organization_vre') +def index(): + group_type = _guess_group_type() + page = h.get_page_number(request.args) or 1 + items_per_page = 21 + context = {'model': model, 'session': model.Session, + 'user': g.user, 'for_view': True, + 'with_private': False} + + q = g.q = request.params.get('q', '') + sort_by = g.sort_by_selected = request.args.get('sort') + + try: + logic.check_access('site_read', context) + logic.check_access('group_list', context) + except NotAuthorized: + abort(403, _('Not authorized to see this page')) + # pass user info to context as needed to view private datasets of + # orgs correctly + + if g.userobj: + context['user_id'] = g.userobj.id + context['user_is_admin'] = g.userobj.sysadmin + + data_dict_global_results = { + 'all_fields': False, + 'q': q, + 'sort': sort_by, + 'type': group_type or 'group', + } + global_results = logic.get_action('group_list')(context,data_dict_global_results) + + data_dict_page_results = { + 'all_fields': True, + 'q': q, + 'sort': sort_by, + 'type': group_type or 'group', + 'limit': items_per_page, + 'offset': items_per_page * (page - 1), + } + page_results = logic.get_action('group_list')(context, data_dict_page_results) + + g.page = h.Page( + collection=global_results, + page=page, + url=h.pager_url, + items_per_page=items_per_page, + ) + g.page.items = page_results + return render_template('organization_vre/index.html', + extra_vars={'group_type': group_type}) + + +@organization_vre.route('/organization_vre/') +def read(id, limit=20): + #group_type = self._ensure_controller_matches_group_type( + # id.split('@')[0]) + group_type = 'organization' + + context = {'model': model, 'session': model.Session, + 'user': g.user, + 'schema': logic.schema.group_form_schema(), + 'for_view': True} + data_dict = {'id': id, 'type': group_type} + + # recupero eventuale query di ricerca + g.q = request.args.get('q', '') + + try: + #i dataset non si includono nel risultato + data_dict['include_datasets'] = False + g.group_dict = logic.get_action('group_show')(context, data_dict) + g.group = context['group'] + except (NotFound, NotAuthorized): + abort(404, _('Group not found')) + + #read(id, limit, group_type) + return render_template('organization_vre/read.html', + extra_vars={'group_type': group_type}) + \ No newline at end of file diff --git a/ckanext/d4science/views_routes/systemtype.py b/ckanext/d4science/views_routes/systemtype.py new file mode 100644 index 0000000..a6a33c7 --- /dev/null +++ b/ckanext/d4science/views_routes/systemtype.py @@ -0,0 +1,73 @@ +import logging +import ckan.plugins as p +from ckan.common import OrderedDict, _ +import ckan.lib.search as search +import ckan.model as model +import ckan.logic as logic +import ckan.lib.helpers as h +from flask import Blueprint, render_template, request, g +from ckan.lib.search import SearchError +from urllib.parse import urlencode + +d4s_type_blueprint = Blueprint('d4s_type', __name__) + +@d4s_type_blueprint.route('/') +def index(): + try: + # package search + context = {'model': model, 'session': model.Session,'user': g.user, 'auth_user_obj': g.userobj} + + facets = OrderedDict() + + default_facet_titles = { + 'organization': _('Organizations'), + 'groups': _('Groups'), + 'tags': _('Tags'), + 'res_format': _('Formats'), + 'license_id': _('Licenses'), + } + + for facet in g.facets: + if facet in default_facet_titles: + facets[facet] = default_facet_titles[facet] + else: + facets[facet] = facet + + # Facet titles + for plugin in p.PluginImplementations(p.IFacets): + facets = plugin.dataset_facets(facets, 'dataset') + + g.facet_titles = facets + + data_dict = { + 'q': '*:*', + 'facet.field': list(facets.keys()), + 'rows': 4, + 'start': 0, + 'sort': 'views_recent desc', + 'fq': 'capacity:"public"' + } + query = logic.get_action('package_search')(context, data_dict) + g.search_facets = query['search_facets'] + g.package_count = query['count'] + g.datasets = query['results'] + + #print "c.search_facets: " + #print " ".join(c.search_facets) + + except search.SearchError: + g.package_count = 0 + + if g.userobj and not g.userobj.email: + #url = h.url_for(controller='user', action='edit') pylons + url = h.url_for('user.edit') + msg = _('Please update your profile' + ' and add your email address. ') % url + \ + _('%s uses your email address' + ' if you need to reset your password.') \ + % g.site_title + h.flash_notice(msg, allow_html=True) + + #return base.render('type/index.html', cache_force=True) pylons + return render_template('type/index.html', cache_force=True) + diff --git a/dev-requirements.txt b/dev-requirements.txt index eac82b4..eba64ef 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1 +1,5 @@ pytest-ckan +webhelpers2==2.1 +xmltodict +pyqrcode +collections \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e69de29..949d55f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,4 @@ +webhelpers2==2.1 +xmltodict +pyqrcode +collections \ No newline at end of file