CatalogueRevamping
/
spatial-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update commands

This commit is contained in:
Sergey Motornyuk 2019-12-11 14:22:28 +02:00
parent 2ab242ed97
commit 2803729263
5 changed files with 208 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
ckanext/spatial/cli.py Normal file
View File

@ -0,0 +1,55 @@
# encoding: utf-8
import click
import logging
from ckan.cli import click_config_option
from ckan.cli.cli import CkanCommand
import ckanext.spatial.util as util
log = logging.getLogger(__name__)
@click.group(short_help=u"Validation commands")
@click.help_option(u"-h", u"--help")
@click_config_option
@click.pass_context
def validation(ctx, config, *args, **kwargs):
ctx.obj = CkanCommand(config)
@validation.command()
@click.argument('pkg', required=False)
def report(pkg):
return util.report(pkg)
@validation.command('report-csv')
@click.argument('filepath')
def report_csv(filepath):
return util.report_csv(filepath)
@validation.command('file')
@click.argument('filepath')
def validate_file(filepath):
return util.validate_file(filepath)
@click.group(short_help=u"Performs spatially related operations.")
@click.help_option(u"-h", u"--help")
@click_config_option
@click.pass_context
def spatial(ctx, config, *args, **kwargs):
ctx.obj = CkanCommand(config)
@spatial.command()
@click.argument('srid', required=False)
def initdb(srid):
return util.initdb(srid)
@spatial.command('extents')
def update_extents():
return util.update_extents()

59
ckanext/spatial/commands/spatial.py
View File

@ -1,11 +1,11 @@
import sys
import re
from pprint import pprint
import logging
import logging
from ckan.lib.cli import CkanCommand
from ckan.lib.helpers import json
from ckanext.spatial.lib import save_package_extent
import ckanext.spatial.util as util
log = logging.getLogger(__name__)
class Spatial(CkanCommand):
@ -20,7 +20,7 @@ class Spatial(CkanCommand):
spatial extents
Creates or updates the extent geometry column for datasets with
an extent defined in the 'spatial' extra.
The commands should be run from the ckanext-spatial directory and expect
a development.ini file to be present. Most of the time you will
specify the config explicitly though::
@ -31,7 +31,7 @@ class Spatial(CkanCommand):
summary = __doc__.split('\n')[0]
usage = __doc__
max_args = 2
max_args = 2
min_args = 0
def command(self):
@ -43,7 +43,7 @@ class Spatial(CkanCommand):
sys.exit(1)
cmd = self.args[0]
if cmd == 'initdb':
self.initdb()
self.initdb()
elif cmd == 'extents':
self.update_extents()
else:
@ -51,47 +51,10 @@ class Spatial(CkanCommand):
def initdb(self):
if len(self.args) >= 2:
srid = unicode(self.args[1])
srid = self.args[1]
else:
srid = None
from ckanext.spatial.model import setup as db_setup
db_setup(srid)
print 'DB tables created'
return util.initdb(srid)
def update_extents(self):
from ckan.model import PackageExtra, Package, Session
conn = Session.connection()
packages = [extra.package \
for extra in \
Session.query(PackageExtra).filter(PackageExtra.key == 'spatial').all()]
errors = []
count = 0
for package in packages:
try:
value = package.extras['spatial']
log.debug('Received: %r' % value)
geometry = json.loads(value)
count += 1
except ValueError,e:
errors.append(u'Package %s - Error decoding JSON object: %s' % (package.id,str(e)))
except TypeError,e:
errors.append(u'Package %s - Error decoding JSON object: %s' % (package.id,str(e)))
save_package_extent(package.id,geometry)
Session.commit()
if errors:
msg = 'Errors were found:\n%s' % '\n'.join(errors)
print msg
msg = "Done. Extents generated for %i out of %i packages" % (count,len(packages))
print msg
return util.update_extents()

76
ckanext/spatial/commands/validation.py
View File

@ -1,13 +1,11 @@
import sys
import re
import os
from pprint import pprint
import logging
from lxml import etree
from ckan.lib.cli import CkanCommand
import ckanext.spatial.util as util
log = logging.getLogger(__name__)
class Validation(CkanCommand):
@ -21,7 +19,7 @@ class Validation(CkanCommand):
validation report-csv <filename>.csv
Performs validation on all the harvested metadata in the db and
writes a report in CSV format to the given filepath.
validation file <filename>.xml
Performs validation on the given metadata file.
'''
@ -48,81 +46,25 @@ class Validation(CkanCommand):
print 'Command %s not recognized' % cmd
def report(self):
from ckan import model
from ckanext.harvest.model import HarvestObject
from ckanext.spatial.lib.reports import validation_report
if len(self.args) >= 2:
package_ref = unicode(self.args[1])
pkg = model.Package.get(package_ref)
if not pkg:
print 'Package ref "%s" not recognised' % package_ref
sys.exit(1)
pkg = self.args[1]
else:
pkg = None
report = validation_report(package_id=pkg.id)
for row in report.get_rows_html_formatted():
print
for i, col_name in enumerate(report.column_names):
print ' %s: %s' % (col_name, row[i])
return util.report(pkg)
def validate_file(self):
from ckanext.spatial.harvesters import SpatialHarvester
from ckanext.spatial.model import ISODocument
if len(self.args) > 2:
print 'Too many parameters %i' % len(self.args)
sys.exit(1)
if len(self.args) < 2:
print 'Not enough parameters %i' % len(self.args)
sys.exit(1)
metadata_filepath = self.args[1]
if not os.path.exists(metadata_filepath):
print 'Filepath %s not found' % metadata_filepath
sys.exit(1)
with open(metadata_filepath, 'rb') as f:
metadata_xml = f.read()
validators = SpatialHarvester()._get_validator()
print 'Validators: %r' % validators.profiles
try:
xml_string = metadata_xml.encode("utf-8")
except UnicodeDecodeError, e:
print 'ERROR: Unicode Error reading file \'%s\': %s' % \
(metadata_filepath, e)
sys.exit(1)
#import pdb; pdb.set_trace()
xml = etree.fromstring(xml_string)
# XML validation
valid, errors = validators.is_valid(xml)
# CKAN read of values
if valid:
try:
iso_document = ISODocument(xml_string)
iso_values = iso_document.read_values()
except Exception, e:
valid = False
errors.append('CKAN exception reading values from ISODocument: %s' % e)
print '***************'
print 'Summary'
print '***************'
print 'File: \'%s\'' % metadata_filepath
print 'Valid: %s' % valid
if not valid:
print 'Errors:'
print pprint(errors)
print '***************'
return util.validate_file(self.args[1])
def report_csv(self):
from ckanext.spatial.lib.reports import validation_report
if len(self.args) != 2:
print 'Wrong number of arguments'
sys.exit(1)
csv_filepath = self.args[1]
report = validation_report()
with open(csv_filepath, 'wb') as f:
f.write(report.get_csv())
return util.report_csv(self.args[1])

133
ckanext/spatial/util.py Normal file
View File

@ -0,0 +1,133 @@
# -*- coding: utf-8 -*-
import os
import sys
import logging
from ckan.lib.helpers import json
from lxml import etree
from pprint import pprint
from ckanext.spatial.lib import save_package_extent
log = logging.getLogger(__name__)
def report(pkg=None):
from ckan import model
from ckanext.harvest.model import HarvestObject
from ckanext.spatial.lib.reports import validation_report
if pkg:
package_ref = unicode(pkg)
pkg = model.Package.get(package_ref)
if not pkg:
print 'Package ref "%s" not recognised' % package_ref
sys.exit(1)
report = validation_report(package_id=pkg.id)
for row in report.get_rows_html_formatted():
print
for i, col_name in enumerate(report.column_names):
print ' %s: %s' % (col_name, row[i])
def validate_file(metadata_filepath):
from ckanext.spatial.harvesters import SpatialHarvester
from ckanext.spatial.model import ISODocument
if not os.path.exists(metadata_filepath):
print 'Filepath %s not found' % metadata_filepath
sys.exit(1)
with open(metadata_filepath, 'rb') as f:
metadata_xml = f.read()
validators = SpatialHarvester()._get_validator()
print 'Validators: %r' % validators.profiles
try:
xml_string = metadata_xml.encode("utf-8")
except UnicodeDecodeError, e:
print 'ERROR: Unicode Error reading file \'%s\': %s' % \
(metadata_filepath, e)
sys.exit(1)
#import pdb; pdb.set_trace()
xml = etree.fromstring(xml_string)
# XML validation
valid, errors = validators.is_valid(xml)
# CKAN read of values
if valid:
try:
iso_document = ISODocument(xml_string)
iso_values = iso_document.read_values()
except Exception, e:
valid = False
errors.append(
'CKAN exception reading values from ISODocument: %s' % e)
print '***************'
print 'Summary'
print '***************'
print 'File: \'%s\'' % metadata_filepath
print 'Valid: %s' % valid
if not valid:
print 'Errors:'
print pprint(errors)
print '***************'
def report_csv(csv_filepath):
from ckanext.spatial.lib.reports import validation_report
report = validation_report()
with open(csv_filepath, 'wb') as f:
f.write(report.get_csv())
def initdb(srid=None):
if srid:
srid = unicode(srid)
from ckanext.spatial.model import setup as db_setup
db_setup(srid)
print 'DB tables created'
def update_extents():
from ckan.model import PackageExtra, Package, Session
conn = Session.connection()
packages = [extra.package \
for extra in \
Session.query(PackageExtra).filter(PackageExtra.key == 'spatial').all()]
errors = []
count = 0
for package in packages:
try:
value = package.extras['spatial']
log.debug('Received: %r' % value)
geometry = json.loads(value)
count += 1
except ValueError, e:
errors.append(u'Package %s - Error decoding JSON object: %s' %
(package.id, str(e)))
except TypeError, e:
errors.append(u'Package %s - Error decoding JSON object: %s' %
(package.id, str(e)))
save_package_extent(package.id, geometry)
Session.commit()
if errors:
msg = 'Errors were found:\n%s' % '\n'.join(errors)
print msg
msg = "Done. Extents generated for %i out of %i packages" % (count,
len(packages))
print msg

1
setup.py
View File

@ -48,7 +48,6 @@ setup(
[console_scripts]
spatial = ckanext.spatial.cli:spatial
ckan-pycsw = ckanext.spatial.cli:ckan_pycsw
validation = ckanext.spatial.cli:validation
""",
)