* first commit for the CKAN 2.10 upgrade * upgrading the MEF part in utils.py * update GN for CKAN 2.10 --------- Co-authored-by: gpetrak <gkpetrak@gmail.com>
This commit is contained in:
Emanuele Tajariol
GitHub
parent
c059f18be9
commit
c5ed4e55d3
|
|
@ -13,12 +13,12 @@ from ckan.plugins.core import SingletonPlugin
|
|||
from ckanext.spatial.lib.csw_client import CswService
|
||||
from ckanext.spatial.harvesters.csw import CSWHarvester
|
||||
|
||||
from ckanext.spatial.model import ISODocument
|
||||
from ckanext.spatial.model import ISOElement
|
||||
from ckanext.spatial.harvested_metadata import ISODocument
|
||||
from ckanext.spatial.harvested_metadata import ISOElement
|
||||
|
||||
from ckan.logic import ValidationError, NotFound, get_action
|
||||
|
||||
from pylons import config
|
||||
from ckan.common import config
|
||||
from datetime import datetime
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -80,7 +80,7 @@ class GeoNetworkHarvester(CSWHarvester, SingletonPlugin):
|
|||
|
||||
existing_keys = [entry.get('key') for entry in package_dict['extras']]
|
||||
|
||||
for key, value in default_extras.iteritems():
|
||||
for key, value in default_extras.items():
|
||||
log.debug('Processing extra %s', key)
|
||||
if not key in existing_keys or override_extras:
|
||||
# Look for replacement strings
|
||||
|
|
@ -163,8 +163,10 @@ class GeoNetworkHarvester(CSWHarvester, SingletonPlugin):
|
|||
version = self.source_config.get('version')
|
||||
client = GeoNetworkClient(gn_localized_url, version)
|
||||
cats = client.retrieveMetadataCategories(harvest_object.guid)
|
||||
log.info(':::::::::::::-TOPIC-CATEGORY-::::::::::::: %r ', cats)
|
||||
|
||||
for cat in cats:
|
||||
log.info('group_mapping %r', group_mapping.items())
|
||||
groupname = group_mapping[cat]
|
||||
|
||||
printname = groupname if not None else "NONE"
|
||||
|
|
@ -172,6 +174,7 @@ class GeoNetworkHarvester(CSWHarvester, SingletonPlugin):
|
|||
|
||||
if groupname:
|
||||
try:
|
||||
log.info('groupname1 %r', groupname)
|
||||
data_dict = {'id': groupname}
|
||||
get_action('group_show')(context, data_dict)
|
||||
#log.info('Group %s found %s' % (groupname, group))
|
||||
|
|
@ -180,10 +183,11 @@ class GeoNetworkHarvester(CSWHarvester, SingletonPlugin):
|
|||
#else:
|
||||
#validated_groups.append(group['id'])
|
||||
validated_groups.append({'name': groupname})
|
||||
except NotFound, e:
|
||||
except NotFound as e:
|
||||
log.warning('Group %s from category %s is not available' % (groupname, cat))
|
||||
except Exception, e:
|
||||
log.warning('Error handling groups for metadata %s' % harvest_object.guid)
|
||||
except Exception as e:
|
||||
# log.warning('Error handling groups for metadata %s' % harvest_object.guid)
|
||||
log.warning('Error handling groups for metadata %r', e)
|
||||
|
||||
return validated_groups
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
import logging
|
||||
#import re
|
||||
import urllib
|
||||
import urllib2
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from StringIO import StringIO
|
||||
import io
|
||||
from lxml import etree
|
||||
|
||||
GEONETWORK_V26 = "2.6"
|
||||
|
|
@ -27,26 +27,29 @@ class GeoNetworkClient(object):
|
|||
def retrieveInfo(self, uuid):
|
||||
|
||||
if self.version == GEONETWORK_V26:
|
||||
url = "%s/srv/en/mef.export" % self.base
|
||||
#headers = {
|
||||
#"Content-Type": "application/x-www-form-urlencoded",
|
||||
#"Accept": "text/plain"
|
||||
#}
|
||||
query = urllib.urlencode({
|
||||
"uuid": uuid
|
||||
})
|
||||
# url = "%s/srv/en/mef.export" % self.base
|
||||
url = "%smef.export?uuid=%s" % (self.base, uuid)
|
||||
|
||||
logger.info('Loading MEF for %s', uuid)
|
||||
request = urllib2.Request(url, query)
|
||||
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(), urllib2.HTTPRedirectHandler())
|
||||
logger.info('URL %r ', url)
|
||||
|
||||
#headers = {
|
||||
# "Content-Type": "application/x-www-form-urlencoded",
|
||||
# "Accept": "text/plain"
|
||||
|
||||
#query = urllib.parse.urlencode({
|
||||
# "uuid": uuid
|
||||
#}).encode('utf-8')
|
||||
|
||||
request = urllib.request.Request(url, method='GET')
|
||||
|
||||
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(), urllib.request.HTTPRedirectHandler())
|
||||
|
||||
response = opener.open(request) # will get a ZIP file
|
||||
content = response.read()
|
||||
|
||||
#logger.info('----> %s', content)
|
||||
|
||||
#print 'RESPONSE ', content
|
||||
|
||||
zdata = StringIO(content)
|
||||
zdata = io.BytesIO(content)
|
||||
zfile = zipfile.ZipFile(zdata)
|
||||
|
||||
xml = None
|
||||
|
|
@ -54,19 +57,20 @@ class GeoNetworkClient(object):
|
|||
for name in zfile.namelist():
|
||||
#logger.info(' MEF entry: %s', name)
|
||||
#print ' MEF entry: ', name
|
||||
if name == 'info.xml':
|
||||
if name == 'metadata.xml':
|
||||
uncompressed = zfile.read(name)
|
||||
xml = etree.fromstring(uncompressed)
|
||||
|
||||
|
||||
return xml
|
||||
|
||||
def retrieveMetadataCategories(self, uuid):
|
||||
xml = self.retrieveInfo(uuid)
|
||||
|
||||
cats = []
|
||||
|
||||
for cat in xml.findall('categories/category'):
|
||||
cats.append(cat.get('name'))
|
||||
for cat in xml.iter('{http://www.isotc211.org/2005/gmd}MD_TopicCategoryCode'):
|
||||
cat = cat.text
|
||||
logger.info('cat %r', cat)
|
||||
cats.append(cat)
|
||||
|
||||
return cats
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue