Merge branch 'docs'
This commit is contained in:
amercader
commit
dc587356c1
|
|
@ -4,3 +4,4 @@ syntax: glob
|
|||
*.swp
|
||||
*~
|
||||
dist
|
||||
build
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
[submodule "doc/_themes/sphinx-theme-okfn"]
|
||||
path = doc/_themes/sphinx-theme-okfn
|
||||
url = https://github.com/okfn/sphinx-theme-okfn.git
|
||||
1065
README.rst
1065
README.rst
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1 @@
|
|||
Sphinx==1.1.3
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 96 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 232 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 90 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 78 KiB |
|
|
@ -0,0 +1,29 @@
|
|||
<p>An <a href="http://okfn.org">Open Knowledge Foundation</a> project.</p>
|
||||
|
||||
<p class="copyright">
|
||||
{%- if show_copyright %}
|
||||
{% trans copyright=copyright|safe|e %}{{ copyright }}{% endtrans %}<br/>
|
||||
{%- endif %}
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<a href="https://github.com/okfn/ckanext-spatial">Source</a>
|
||||
—
|
||||
<a href="https://github.com/okfn/ckanext-spatial/issues">Issues</a>
|
||||
—
|
||||
<a href="http://lists.okfn.org/mailman/listinfo/ckan-dev">Mailing List</a>
|
||||
—
|
||||
<a href="http://twitter.com/CKANProject">Twitter @CKANProject</a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
Related Projects:
|
||||
<a href="http://thedatahub.org/">The DataHub</a>
|
||||
—
|
||||
<a href="http://datacatalogs.org">DataCatalogs.org</a>
|
||||
—
|
||||
<a href="http://openspending.org">OpenSpending.org</a>
|
||||
—
|
||||
<a href="http://opendatahandbook.org">Open Data Handbook</a>
|
||||
</p>
|
||||
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 59688a6679f3373a57e8d4e60e43f1b249878eb3
|
||||
|
|
@ -0,0 +1,266 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# ckanext-spatial documentation build configuration file, created by
|
||||
# sphinx-quickstart on Wed Apr 10 17:17:12 2013.
|
||||
#
|
||||
# This file is execfile()d with the current directory set to its containing dir.
|
||||
#
|
||||
# Note that not all possible configuration values are present in this
|
||||
# autogenerated file.
|
||||
#
|
||||
# All configuration values have a default; values that are commented out
|
||||
# serve to show the default.
|
||||
|
||||
import sys, os
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#sys.path.insert(0, os.path.abspath('.'))
|
||||
|
||||
# -- General configuration -----------------------------------------------------
|
||||
|
||||
# If your documentation needs a minimal Sphinx version, state it here.
|
||||
#needs_sphinx = '1.0'
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be extensions
|
||||
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.intersphinx']
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
# The suffix of source filenames.
|
||||
source_suffix = '.rst'
|
||||
|
||||
# The encoding of source files.
|
||||
#source_encoding = 'utf-8-sig'
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
project = u'ckanext-spatial'
|
||||
copyright = u'2013, Open Knowledge Foundation'
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
version = '0.1'
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = '0.1'
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
#language = None
|
||||
|
||||
# There are two options for replacing |today|: either, you set today to some
|
||||
# non-false value, then it is used:
|
||||
#today = ''
|
||||
# Else, today_fmt is used as the format for a strftime call.
|
||||
#today_fmt = '%B %d, %Y'
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
exclude_patterns = []
|
||||
|
||||
# The reST default role (used for this markup: `text`) to use for all documents.
|
||||
#default_role = None
|
||||
|
||||
# If true, '()' will be appended to :func: etc. cross-reference text.
|
||||
#add_function_parentheses = True
|
||||
|
||||
# If true, the current module name will be prepended to all description
|
||||
# unit titles (such as .. function::).
|
||||
#add_module_names = True
|
||||
|
||||
# If true, sectionauthor and moduleauthor directives will be shown in the
|
||||
# output. They are ignored by default.
|
||||
#show_authors = False
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = 'sphinx'
|
||||
|
||||
# A list of ignored prefixes for module index sorting.
|
||||
#modindex_common_prefix = []
|
||||
|
||||
# If true, keep warnings as "system message" paragraphs in the built documents.
|
||||
#keep_warnings = False
|
||||
|
||||
exclude_trees = ['.build']
|
||||
|
||||
|
||||
# -- Options for HTML output ---------------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#html_theme = 'default'
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
#html_theme_options = {}
|
||||
|
||||
# Add any paths that contain custom themes here, relative to this directory.
|
||||
#html_theme_path = []
|
||||
sys.path.append(os.path.abspath('_themes'))
|
||||
html_theme_path = ['_themes']
|
||||
html_theme = 'sphinx-theme-okfn'
|
||||
html_theme_options = {
|
||||
'logo_icon': 'ckanlogo.png',
|
||||
'show_version': False,
|
||||
'show_okfn_logo': False,
|
||||
}
|
||||
html_sidebars = {
|
||||
'**': ['globaltoc.html']
|
||||
}
|
||||
|
||||
|
||||
# The name for this set of Sphinx documents. If None, it defaults to
|
||||
# "<project> v<release> documentation".
|
||||
#html_title = None
|
||||
|
||||
# A shorter title for the navigation bar. Default is the same as html_title.
|
||||
#html_short_title = None
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top
|
||||
# of the sidebar.
|
||||
#html_logo = None
|
||||
|
||||
# The name of an image file (within the static path) to use as favicon of the
|
||||
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
|
||||
# pixels large.
|
||||
#html_favicon = None
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
|
||||
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
|
||||
# using the given strftime format.
|
||||
#html_last_updated_fmt = '%b %d, %Y'
|
||||
|
||||
# If true, SmartyPants will be used to convert quotes and dashes to
|
||||
# typographically correct entities.
|
||||
#html_use_smartypants = True
|
||||
|
||||
# Custom sidebar templates, maps document names to template names.
|
||||
#html_sidebars = {}
|
||||
|
||||
# Additional templates that should be rendered to pages, maps page names to
|
||||
# template names.
|
||||
#html_additional_pages = {}
|
||||
|
||||
# If false, no module index is generated.
|
||||
#html_domain_indices = True
|
||||
|
||||
# If false, no index is generated.
|
||||
#html_use_index = True
|
||||
|
||||
# If true, the index is split into individual pages for each letter.
|
||||
#html_split_index = False
|
||||
|
||||
# If true, links to the reST sources are added to the pages.
|
||||
#html_show_sourcelink = True
|
||||
|
||||
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
|
||||
#html_show_sphinx = True
|
||||
|
||||
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
|
||||
#html_show_copyright = True
|
||||
|
||||
# If true, an OpenSearch description file will be output, and all pages will
|
||||
# contain a <link> tag referring to it. The value of this option must be the
|
||||
# base URL from which the finished HTML is served.
|
||||
#html_use_opensearch = ''
|
||||
|
||||
# This is the file name suffix for HTML files (e.g. ".xhtml").
|
||||
#html_file_suffix = None
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'ckanext-spatialdoc'
|
||||
|
||||
|
||||
# -- Options for LaTeX output --------------------------------------------------
|
||||
|
||||
latex_elements = {
|
||||
# The paper size ('letterpaper' or 'a4paper').
|
||||
#'papersize': 'letterpaper',
|
||||
|
||||
# The font size ('10pt', '11pt' or '12pt').
|
||||
#'pointsize': '10pt',
|
||||
|
||||
# Additional stuff for the LaTeX preamble.
|
||||
#'preamble': '',
|
||||
}
|
||||
|
||||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title, author, documentclass [howto/manual]).
|
||||
latex_documents = [
|
||||
('index', 'ckanext-spatial.tex', u'ckanext-spatial Documentation',
|
||||
u'Open Knowledge Foundation', 'manual'),
|
||||
]
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
# the title page.
|
||||
#latex_logo = None
|
||||
|
||||
# For "manual" documents, if this is true, then toplevel headings are parts,
|
||||
# not chapters.
|
||||
#latex_use_parts = False
|
||||
|
||||
# If true, show page references after internal links.
|
||||
#latex_show_pagerefs = False
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
#latex_show_urls = False
|
||||
|
||||
# Documents to append as an appendix to all manuals.
|
||||
#latex_appendices = []
|
||||
|
||||
# If false, no module index is generated.
|
||||
#latex_domain_indices = True
|
||||
|
||||
|
||||
# -- Options for manual page output --------------------------------------------
|
||||
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
('index', 'ckanext-spatial', u'ckanext-spatial Documentation',
|
||||
[u'Open Knowledge Foundation'], 1)
|
||||
]
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
#man_show_urls = False
|
||||
|
||||
|
||||
# -- Options for Texinfo output ------------------------------------------------
|
||||
|
||||
# Grouping the document tree into Texinfo files. List of tuples
|
||||
# (source start file, target name, title, author,
|
||||
# dir menu entry, description, category)
|
||||
texinfo_documents = [
|
||||
('index', 'ckanext-spatial', u'ckanext-spatial Documentation',
|
||||
u'Open Knowledge Foundation', 'ckanext-spatial', 'One line description of project.',
|
||||
'Miscellaneous'),
|
||||
]
|
||||
|
||||
# Documents to append as an appendix to all manuals.
|
||||
#texinfo_appendices = []
|
||||
|
||||
# If false, no module index is generated.
|
||||
#texinfo_domain_indices = True
|
||||
|
||||
# How to display URL addresses: 'footnote', 'no', or 'inline'.
|
||||
#texinfo_show_urls = 'footnote'
|
||||
|
||||
# If true, do not generate a @detailmenu in the "Top" node's menu.
|
||||
#texinfo_no_detailmenu = False
|
||||
|
||||
|
||||
# Example configuration for intersphinx: refer to the Python standard library.
|
||||
intersphinx_mapping = {'http://docs.python.org/': None}
|
||||
|
|
@ -0,0 +1,260 @@
|
|||
===========
|
||||
CSW support
|
||||
===========
|
||||
|
||||
The extension provides the support for the CSW_ standard, a specification from
|
||||
the Open Geospatial Consortium for exposing geospatial catalogues over the web.
|
||||
|
||||
This support consists of:
|
||||
|
||||
* Ability to import records from CSW servers with the CSW harvester. See
|
||||
:doc:`harvesters` for more details.
|
||||
|
||||
* Integration with pycsw_ to provide a fully compliat CSW interface for
|
||||
harvested records. This integration is described on the following sections.
|
||||
|
||||
|
||||
ckan-pycsw
|
||||
----------
|
||||
|
||||
The spatial extension offers the ``ckan-pycsw`` command, which allows to expose
|
||||
the spatial datasets harvested from other sources in a CSW interface. This is
|
||||
powered by pycsw_, which fully implements the OGC CSW specification.
|
||||
|
||||
How it works
|
||||
++++++++++++
|
||||
|
||||
|
||||
The current implementation is based on CKAN and pycsw being loosely integrated
|
||||
via the CKAN API. pycsw will be generally installed in the same server as CKAN
|
||||
(although it can also be run on a separate one), and the synchronization
|
||||
command will be run regularly to keep the records on the pycsw repository up to
|
||||
date. This is done using the CKAN API to get all the datasets identifiers (more
|
||||
precisely the ones from datasets that have been harvested) and then deciding
|
||||
which ones need to be created, updated or deleted on the pycsw repository. For
|
||||
those that need to be created or updated, the original harvested spatial
|
||||
document (ie ISO 19139) is requested from CKAN, and it is then imported using
|
||||
pycsw internal functions::
|
||||
|
||||
Harvested
|
||||
datasets
|
||||
+
|
||||
|
|
||||
v
|
||||
+--------+ +---------+
|
||||
| | CKAN API | |
|
||||
| CKAN | +------------> | pycsw | +------> CSW
|
||||
| | | |
|
||||
+--------+ +---------+
|
||||
|
||||
|
||||
Remember, only datasets that were harvested with the :doc:`harvesters`
|
||||
can currently be exposed via pycsw.
|
||||
|
||||
All necessary tasks are done with the ``ckan-pycsw`` command. To get more
|
||||
details of its usage, run the following::
|
||||
|
||||
cd /usr/lib/ckan/default/src/ckanext-spatial
|
||||
paster ckan-pycsw --help
|
||||
|
||||
|
||||
Setup
|
||||
+++++
|
||||
|
||||
1. Install pycsw. There are several options for this, depending on your
|
||||
server setup, check the `pycsw documentation`_.
|
||||
|
||||
.. note:: CKAN integration requires at least pycsw version 1.6.1. Make sure
|
||||
to install at least this version.
|
||||
|
||||
The following instructions assume that you have installed CKAN via a
|
||||
`package install`_ and should be run as root, but the steps are the same if
|
||||
you are setting it up in another location::
|
||||
|
||||
cd /usr/lib/ckan/default/src
|
||||
source ../bin/activate
|
||||
|
||||
# From now on the virtualenv should be activated
|
||||
|
||||
git clone https://github.com/geopython/pycsw.git
|
||||
cd pycsw
|
||||
# Remember to use at least pycsw 1.6.1
|
||||
git checkout 1.6.1
|
||||
pip install -e .
|
||||
python setup.py build
|
||||
python setup.py install
|
||||
|
||||
2. Create a database for pycsw. In theory you can use the same database that
|
||||
CKAN is using, but if you want to keep them separated, use the following
|
||||
command to create a new one (we'll use the same default user though)::
|
||||
|
||||
sudo -u postgres createdb -O ckan_default pycsw -E utf-8
|
||||
|
||||
It is strongly recommended that you install PostGIS in the pycsw databaset,
|
||||
so its spatial functions are used. See the :ref:`install_postgis`
|
||||
section for details.
|
||||
|
||||
3. Configure pycsw. An example configuration file is included on the source::
|
||||
|
||||
cp default-sample.cfg default.cfg
|
||||
|
||||
To keep things tidy we will create a symlink to this file on the CKAN
|
||||
configuration directory::
|
||||
|
||||
ln -s /usr/lib/ckan/default/src/pycsw/default.cfg /etc/ckan/default/pycsw.cfg
|
||||
|
||||
Open the file with your favourite editor. The main settings you should tweak
|
||||
are ``server.home`` and ``repository.database``::
|
||||
|
||||
[server]
|
||||
home=/usr/lib/ckan/default/src/pycsw
|
||||
...
|
||||
[repository]
|
||||
database=postgresql://ckan_default:pass@localhost/pycsw
|
||||
|
||||
The rest of the options are described `here <http://pycsw.org/docs/configuration.html>`_.
|
||||
|
||||
4. Setup the pycsw table. This is done with the ``ckan-pycsw`` paster command
|
||||
(Remember to have the virtualenv activated when running it)::
|
||||
|
||||
cd /usr/lib/ckan/default/src/ckanext-spatial
|
||||
paster ckan-pycsw setup -p /etc/ckan/default/pycsw.cfg
|
||||
|
||||
At this point you should be ready to run pycsw with the wsgi script that it
|
||||
includes::
|
||||
|
||||
cd /usr/lib/ckan/default/src/pycsw
|
||||
python csw.wsgi
|
||||
|
||||
This will run pycsw at http://localhost:8000. Visiting the following URL
|
||||
should return you the Capabilities file:
|
||||
|
||||
http://localhost:8000/?service=CSW&version=2.0.2&request=GetCapabilities
|
||||
|
||||
5. Load the CKAN datasets into pycsw. Again, we will use the ``ckan-pycsw``
|
||||
command for this::
|
||||
|
||||
cd /usr/lib/ckan/default/src/ckanext-spatial
|
||||
paster ckan-pycsw load -p /etc/ckan/default/pycsw.cfg
|
||||
|
||||
.. note:: If you get errors similar to this one, this is caused by
|
||||
limitations on the pycsw model definition. This should be fixed in
|
||||
future versions of pycsw::
|
||||
|
||||
ERROR: not inserted f8d48eaf-780b-40b8-a502-7a903fde5b1c Error:ERROR: value too long for type character varying(256)
|
||||
|
||||
|
||||
When the loading is finished, check that results are returned when visiting
|
||||
this link:
|
||||
|
||||
http://localhost:8000/?request=GetRecords&service=CSW&version=2.0.2&resultType=results&outputSchema=http://www.isotc211.org/2005/gmd&typeNames=csw:Record&elementSetName=summary
|
||||
|
||||
The ``numberOfRecordsMatched`` should match the number of harvested datasets
|
||||
in CKAN (minus import errors). If you run the command again new or udpated
|
||||
datasets will be synchronized and deleted datasets from CKAN will be removed
|
||||
from pycsw as well.
|
||||
|
||||
Running it on production site
|
||||
+++++++++++++++++++++++++++++
|
||||
|
||||
On a production site you probably want to run the load command regularly to
|
||||
keep CKAN and pycsw in sync, and serve pycsw with Apache + mod_wsgi like CKAN.
|
||||
|
||||
* To run the load command regularly you can set up a cron job. Type ``crontab -e``
|
||||
and copy the following lines::
|
||||
|
||||
# m h dom mon dow command
|
||||
0 * * * * /usr/lib/ckan/default/bin/paster --plugin=ckanext-spatial ckan-pycsw load -p /etc/ckan/default/pycsw.cfg
|
||||
|
||||
This particular example will run the load command every hour. You can of
|
||||
course modify this periodicity, for instance reducing it for huge instances.
|
||||
This `Wikipedia page <http://en.wikipedia.org/wiki/Cron#CRON_expression>`_
|
||||
has a good overview of the crontab syntax.
|
||||
|
||||
* To run pycsw under Apache check the pycsw `installation documentation <http://pycsw.org/docs/installation.html#running-on-wsgi>`_
|
||||
or follow this quick steps (they assume the paths used on the previous steps):
|
||||
|
||||
- Edit ``/etc/apache2/sites-available/ckan_default`` and add the following
|
||||
line just before the existing ``WSGIScriptAlias`` directive::
|
||||
|
||||
WSGIScriptAlias /csw /usr/lib/ckan/default/src/pycsw/csw.wsgi
|
||||
|
||||
- Edit the ``/usr/lib/ckan/default/src/pycsw/csw.wsgi`` file and add these two
|
||||
lines just after the imports on the top of the file::
|
||||
|
||||
activate_this = os.path.join('/usr/lib/ckan/default/bin/activate_this.py')
|
||||
execfile(activate_this, {"__file__":activate_this})
|
||||
|
||||
We need these to activate the virtualenv where we installed pycsw into.
|
||||
|
||||
- Restart Apache::
|
||||
|
||||
service apache2 restart
|
||||
|
||||
pycsw should be now accessible at http://localhost/csw
|
||||
|
||||
|
||||
Legacy plugins and libraries
|
||||
----------------------------
|
||||
|
||||
|
||||
Old CSW Server
|
||||
++++++++++++++
|
||||
|
||||
.. warning:: **Deprecated:** The old csw plugin has been deprecated, please see `ckan-pycsw`_
|
||||
for details on how to integrate with pycsw.
|
||||
|
||||
To activate it, add the ``csw_server`` plugin to your ini file.
|
||||
|
||||
Only harvested datasets are served by this CSW Server. This is because
|
||||
the harvested document is the one that is served, not something derived
|
||||
from the CKAN Dataset object. Datasets that are created in CKAN by methods
|
||||
other than harvesting are not served.
|
||||
|
||||
The currently supported methods with this CSW Server are:
|
||||
* GetCapabilities
|
||||
* GetRecords
|
||||
* GetRecordById
|
||||
|
||||
For example you can ask the capabilities of the CSW server installed into CKAN
|
||||
running on 127.0.0.1:5000 like this::
|
||||
|
||||
curl 'http://127.0.0.1:5000/csw?request=GetCapabilities&service=CSW&version=2.0.2'
|
||||
|
||||
And get a list of the records like this::
|
||||
|
||||
curl 'http://127.0.0.1:5000/csw?request=GetRecords&service=CSW&resultType=results&elementSetName=full&version=2.0.2'
|
||||
|
||||
The standard CSW response is in XML format.
|
||||
|
||||
cswinfo
|
||||
+++++++
|
||||
|
||||
The command-line tool ``cswinfo`` allows to make queries on CSW servers and
|
||||
returns the info in nicely formatted JSON. This may be more convenient to type
|
||||
than using, for example, curl.
|
||||
|
||||
Currently available queries are:
|
||||
* getcapabilities
|
||||
* getidentifiers
|
||||
* getrecords
|
||||
* getrecordbyid
|
||||
|
||||
For details, type::
|
||||
|
||||
cswinfo csw -h
|
||||
|
||||
There are options for querying by only certain types, keywords and typenames
|
||||
as well as configuring the ElementSetName.
|
||||
|
||||
The equivalent example to the one above for asking the cabailities is::
|
||||
|
||||
$ cswinfo csw getcapabilities http://127.0.0.1:5000/csw
|
||||
|
||||
OWSLib is the library used to actually perform the queries.
|
||||
|
||||
.. _pycsw: http://pycsw.org
|
||||
.. _pycsw documentation: http://pycsw.org/docs/installation.html
|
||||
.. _package install: http://docs.ckan.org/en/latest/install-from-package.html
|
||||
.. _CSW: http://www.opengeospatial.org/standards/cat
|
||||
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
==================
|
||||
Spatial Harvesters
|
||||
==================
|
||||
|
||||
Overview and Configuration
|
||||
--------------------------
|
||||
|
||||
The spatial extension provides some harvesters for importing ISO19139-based
|
||||
metadata into CKAN, as well as providing a base class for writing new ones.
|
||||
The harvesters use the interface provided by ckanext-harvest_, so you will need
|
||||
to install and set it up first.
|
||||
|
||||
Once ckanext-harvest is installed, you can add the following plugins to your
|
||||
ini file to enable the different harvesters (If you are upgrading from a
|
||||
previous version to CKAN 2.0 see legacy_harvesters_):
|
||||
|
||||
* ``csw_harvester`` - CSW server
|
||||
* ``waf_harvester`` - WAF (Web Accessible Folder): An online accessible index
|
||||
page with links to metadata documents
|
||||
* ``doc_harvester`` - A single online accessible metadata document.
|
||||
|
||||
Have a look at the `ckanext-harvest documentation`_ if you want to have an
|
||||
overview of how the CKAN harvesters work, but basically there are three
|
||||
separate stages:
|
||||
|
||||
* gather_stage - Aggregates all the remote identifiers for a particular source
|
||||
(eg identifiers for a CSW server, files for a WAF).
|
||||
* fetch_stage - Fetches all the remote documents and stores them on the
|
||||
database.
|
||||
* import_stage - Performs all the processing for transforming the remote
|
||||
content into a CKAN dataset: validates the document, parses it, converts it
|
||||
to a CKAN dataset dict and saves it in the database.
|
||||
|
||||
The extension provides different XSD and schematron based validators. You can
|
||||
specify which validators to use for the remote documents with the following
|
||||
configuration option::
|
||||
|
||||
ckan.spatial.validator.profiles = iso19193eden
|
||||
|
||||
By default, the import stage will stop if the validation of the harvested
|
||||
document fails. This can be modified setting the
|
||||
``ckanext.spatial.harvest.continue_on_validation_errors`` to True. The setting
|
||||
can also be applied at the source level setting to True the
|
||||
``continue_on_validation_errors`` key on the source configuration object.
|
||||
|
||||
By default the harvesting actions (eg creating or updating datasets) will be
|
||||
performed by the internal site admin user. This is the recommended setting,
|
||||
but if necessary, it can be overridden with the
|
||||
``ckanext.spatial.harvest.user_name`` config option, eg to support the old
|
||||
hardcoded 'harvest' user::
|
||||
|
||||
ckanext.spatial.harvest.user_name = harvest
|
||||
|
||||
Customizing the harvesters
|
||||
--------------------------
|
||||
|
||||
The default harvesters provided in this extension can be overriden from
|
||||
extensions to customize to your needs. You can either extend ``CswHarvester``,
|
||||
``WAFfHarverster`` or the main ``SpatialHarvester`` class. There are some
|
||||
extension points that can be safely overriden from your extension. Probably the
|
||||
most useful is ``get_package_dict``, which allows to tweak the dataset fields
|
||||
before creating or updating them. ``transform_to_iso`` allows to hook into
|
||||
transformation mechanisms to transform other formats into ISO1939, the only one
|
||||
directly supported byt he spatial harvesters. Finally, the whole
|
||||
``import_stage`` can be overriden if the default logic does not suit your
|
||||
needs.
|
||||
|
||||
Check the source code of ``ckanext/spatial/harvesters/base.py`` for more
|
||||
details on these functions.
|
||||
|
||||
The `ckanext-geodatagov`_ extension contains live examples on how to extend
|
||||
the default spatial harvesters and create new ones for other spatial services
|
||||
like ArcGIS REST APIs.
|
||||
|
||||
|
||||
Harvest Metadata API
|
||||
--------------------
|
||||
|
||||
This plugin allows to access the actual harvested document via API requests.
|
||||
It is enabled with the following plugin::
|
||||
|
||||
ckan.plugins = spatial_harvest_metadata_api
|
||||
|
||||
(It was previously known as ``inspire_api``)
|
||||
|
||||
To view the harvest objects (containing the harvested metadata) in the web
|
||||
interface, these controller locations are added:
|
||||
|
||||
* raw XML document: /harvest/object/{id}
|
||||
* HTML representation: /harvest/object/{id}/html
|
||||
|
||||
.. note:: The old URLs are now deprecated and redirect to the previously
|
||||
mentioned:
|
||||
|
||||
* /api/2/rest/harvestobject/<id>/xml
|
||||
* /api/2/rest/harvestobject/<id>/html
|
||||
|
||||
|
||||
For those harvest objects that have an original document (which was transformed
|
||||
to ISO), this can be accessed via:
|
||||
|
||||
* raw XML document: /harvest/object/{id}/original
|
||||
* HTML representation: /harvest/object/{id}/html/original
|
||||
|
||||
The HTML representation is created via an XSLT transformation. The extension
|
||||
provides an XSLT file that should work on ISO 19139 based documents, but if you
|
||||
want to use your own on your extension, you can override it using the following
|
||||
configuration options::
|
||||
|
||||
ckanext.spatial.harvest.xslt_html_content = ckanext.myext:templates/xslt/custom.xslt
|
||||
ckanext.spatial.harvest.xslt_html_content_original = ckanext.myext:templates/xslt/custom2.xslt
|
||||
|
||||
If your project does not transform different metadata types you can ignore the
|
||||
second option.
|
||||
|
||||
.. _legacy_harvesters:
|
||||
|
||||
Legacy harvesters
|
||||
-----------------
|
||||
|
||||
Prior to CKAN 2.0, the spatial harvesters available on this extension were
|
||||
based on the GEMINI2 format, an ISO19139 profile used by the UK Location
|
||||
Programme, and the logic for creating or updating datasets and the resulting
|
||||
fields were somehow adapted to the needs for this particular project. The
|
||||
harvesters were still generic enough and should work fine with other ISO19139
|
||||
based sources, but extra care has been put to make the new harvesters more
|
||||
generic and robust, so these ones should only be used on existing instances:
|
||||
|
||||
* ``gemini_csw_harvester``
|
||||
* ``gemini_waf_harvester``
|
||||
* ``gemini_doc_harvester``
|
||||
|
||||
If you are using these harvesters please consider upgrading to the new
|
||||
versions described on the previous section.
|
||||
|
||||
|
||||
.. todo:: Validation library details
|
||||
|
||||
|
||||
.. _ckanext-harvest: https://github.com/okfn/ckanext-harvest
|
||||
.. _ckanext-harvest documentation: https://github.com/okfn/ckanext-harvest#the-harvesting-interface
|
||||
.. _ckanext-geodatagov: https://github.com/okfn/ckanext-geodatagov/blob/master/ckanext/geodatagov/harvesters/
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
==============================================
|
||||
ckanext-spatial - Geo related plugins for CKAN
|
||||
==============================================
|
||||
|
||||
This extension contains plugins that add geospatial capabilities to CKAN_.
|
||||
|
||||
You should have a CKAN instance installed before adding these plugins. Head to
|
||||
the `CKAN documentation`_ for information on how to set up CKAN.
|
||||
|
||||
The extension adds a spatial field to the default CKAN dataset schema,
|
||||
using PostGIS_ as the backend. This allows to perform spatial queries and
|
||||
display the dataset extent on the frontend. It also provides harvesters to
|
||||
import geospatial metadata into CKAN from other sources, as well as commands
|
||||
to support the CSW standard. Finally, it also includes plugins to preview
|
||||
spatial formats such as GeoJSON_.
|
||||
|
||||
|
||||
Contents:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
install
|
||||
spatial-search
|
||||
harvesters
|
||||
csw
|
||||
previews
|
||||
|
||||
.. _CKAN: http://ckan.org
|
||||
.. _CKAN Documentation: http://docs.ckan.org
|
||||
.. _PostGIS: http://postgis.org
|
||||
.. _GeoJSON: http://geojson.org
|
||||
|
||||
|
|
@ -0,0 +1,243 @@
|
|||
======================
|
||||
Installation and Setup
|
||||
======================
|
||||
|
||||
Check the Troubleshooting_ section if you get errors at any stage.
|
||||
|
||||
.. _install_postgis:
|
||||
|
||||
Install PostGIS and system packages
|
||||
-----------------------------------
|
||||
|
||||
.. note:: If you *only* want to load the :doc:`previews` you don't need to
|
||||
install any of the packages on this section and can skip to the
|
||||
next one.
|
||||
|
||||
.. note:: The package names and paths shown are the defaults on an Ubuntu
|
||||
12.04 install (PostgreSQL 9.1 and PostGIS 1.5). Adjust the
|
||||
package names and the paths if you are using a different version of
|
||||
any of them.
|
||||
|
||||
All commands assume an existing CKAN database named ``ckan_default``.
|
||||
|
||||
|
||||
#. Install PostGIS::
|
||||
|
||||
sudo apt-get install postgresql-9.1-postgis
|
||||
|
||||
#. Run the following commands. The first one will create the necessary
|
||||
tables and functions in the database, and the second will populate
|
||||
the spatial reference table::
|
||||
|
||||
sudo -u postgres psql -d ckan_default -f /usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql
|
||||
sudo -u postgres psql -d ckan_default -f /usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql
|
||||
|
||||
.. note:: If using PostgreSQL 8.x, run the following command to enable
|
||||
the necessary language::
|
||||
|
||||
sudo -u postgres createlang plpgsql ckan_default
|
||||
|
||||
#. Change the owner to spatial tables to the CKAN user to avoid errors later
|
||||
on::
|
||||
|
||||
ALTER TABLE spatial_ref_sys OWNER TO ckan_default;
|
||||
ALTER TABLE geometry_columns OWNER TO ckan_default;
|
||||
|
||||
#. Execute the following command to see if PostGIS was properly
|
||||
installed::
|
||||
|
||||
sudo -u postgres psql -d ckan_default -c "SELECT postgis_full_version()"
|
||||
|
||||
You should get something like::
|
||||
|
||||
postgis_full_version
|
||||
------------------------------------------------------------------------------------------------------
|
||||
POSTGIS="1.5.2" GEOS="3.2.2-CAPI-1.6.2" PROJ="Rel. 4.7.1, 23 September 2009" LIBXML="2.7.7" USE_STATS
|
||||
(1 row)
|
||||
|
||||
|
||||
#. Install some other packages needed by the extension dependencies::
|
||||
|
||||
sudo apt-get install python-dev libxml2-dev libxslt1-dev libgeos-c1
|
||||
|
||||
|
||||
Install the extension
|
||||
---------------------
|
||||
|
||||
1. Install this extension into your python environment (where CKAN is also
|
||||
installed).
|
||||
|
||||
.. note:: Depending on the CKAN core version you are targeting you will need
|
||||
to use a different branch from the extension.
|
||||
|
||||
For a production site, use the ``stable`` branch, unless there is a specific
|
||||
branch that targets the CKAN core version that you are using.
|
||||
|
||||
To target the latest CKAN core release::
|
||||
|
||||
(pyenv) $ pip install -e git+https://github.com/okfn/ckanext-spatial.git@stable#egg=ckanext-spatial
|
||||
|
||||
To target an old release (if a release branch exists, otherwise use
|
||||
``stable``)::
|
||||
|
||||
(pyenv) $ pip install -e git+https://github.com/okfn/ckanext-spatial.git@release-v1.8#egg=ckanext-spatial
|
||||
|
||||
To target CKAN ``master``, use the extension ``master`` branch (ie no
|
||||
branch defined)::
|
||||
|
||||
(pyenv) $ pip install -e git+https://github.com/okfn/ckanext-spatial.git#egg=ckanext-spatial
|
||||
|
||||
|
||||
2. Install the rest of python modules required by the extension::
|
||||
|
||||
(pyenv) $ pip install -r pip-requirements.txt
|
||||
|
||||
To use the :doc:`harvesters`, you will need to install and configure the
|
||||
harvester extension: `ckanext-harvest`_. Follow the install instructions on
|
||||
its documentation for details on how to set it up.
|
||||
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
|
||||
Once PostGIS is installed and configured in the database the extension needs
|
||||
to create a table to store the datasets extent, called ``package_extent``.
|
||||
|
||||
This will happen automatically the next CKAN is restarted after adding the
|
||||
plugins on the configuration ini file (eg when restarting Apache).
|
||||
|
||||
If for some reason you need to explicitly create the table beforehand, you can
|
||||
do it with the following command (with the virtualenv activated)::
|
||||
|
||||
(pyenv) $ paster --plugin=ckanext-spatial spatial initdb [srid] --config=mysite.ini
|
||||
|
||||
You can define the SRID of the geometry column. Default is 4326. If you are not
|
||||
familiar with projections, we recommend to use the default value. To know more
|
||||
about PostGIS tables, see :doc:`postgis-manual`
|
||||
|
||||
Each plugin can be enabled by adding its name to the ``ckan.plugins`` in the
|
||||
CKAN ini file. For example::
|
||||
|
||||
ckan.plugins = spatial_metadata spatial_query
|
||||
|
||||
When enabling the spatial metadata, you can define the projection in which
|
||||
extents are stored in the database with the following option. Use the EPSG code
|
||||
as an integer (e.g 4326, 4258, 27700, etc). It defaults to 4326::
|
||||
|
||||
ckan.spatial.srid = 4326
|
||||
|
||||
|
||||
Troubleshooting
|
||||
---------------
|
||||
|
||||
Here are some common problems you may find when installing or using the
|
||||
extension:
|
||||
|
||||
When initializing the spatial tables
|
||||
++++++++++++++++++++++++++++++++++++
|
||||
|
||||
::
|
||||
|
||||
LINE 1: SELECT AddGeometryColumn('package_extent','the_geom', E'4326...
|
||||
^
|
||||
HINT: No function matches the given name and argument types. You might need to add explicit type casts.
|
||||
"SELECT AddGeometryColumn('package_extent','the_geom', %s, 'GEOMETRY', 2)" ('4326',)
|
||||
|
||||
|
||||
PostGIS was not installed correctly. Please check the "Setting up PostGIS"
|
||||
section.
|
||||
|
||||
::
|
||||
|
||||
sqlalchemy.exc.ProgrammingError: (ProgrammingError) permission denied for relation spatial_ref_sys
|
||||
|
||||
|
||||
The user accessing the ckan database needs to be owner (or have permissions)
|
||||
of the geometry_columns and spatial_ref_sys tables.
|
||||
|
||||
When migrating to an existing PostGIS database
|
||||
++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
If you are loading a database dump to an existing PostGIS database, you may
|
||||
find errors like ::
|
||||
|
||||
ERROR: type "spheroid" already exists
|
||||
|
||||
This means that the PostGIS functions are installed, but you may need to
|
||||
create the necessary tables anyway. You can force psql to ignore these
|
||||
errors and continue the transaction with the ON_ERROR_ROLLBACK=on::
|
||||
|
||||
sudo -u postgres psql -d ckan_default -f /usr/share/postgresql/8.4/contrib/postgis-1.5/postgis.sql -v ON_ERROR_ROLLBACK=on
|
||||
|
||||
You will still need to populate the spatial_ref_sys table and change the
|
||||
tables permissions. Refer to the previous section for details on how to do
|
||||
it.
|
||||
|
||||
When performing a spatial query
|
||||
+++++++++++++++++++++++++++++++
|
||||
|
||||
::
|
||||
|
||||
InvalidRequestError: SQL expression, column, or mapped entity expected - got '<class 'ckanext.spatial.model.PackageExtent'>'
|
||||
|
||||
The spatial model has not been loaded. You probably forgot to add the
|
||||
``spatial_metadata`` plugin to your ini configuration file.
|
||||
|
||||
::
|
||||
|
||||
InternalError: (InternalError) Operation on two geometries with different SRIDs
|
||||
|
||||
The spatial reference system of the database geometry column and the one
|
||||
used by CKAN differ. Remember, if you are using a different spatial
|
||||
reference system from the default one (WGS 84 lat/lon, EPSG:4326), you must
|
||||
define it in the configuration file as follows::
|
||||
|
||||
ckan.spatial.srid = 4258
|
||||
|
||||
When running the spatial harvesters
|
||||
+++++++++++++++++++++++++++++++++++
|
||||
|
||||
::
|
||||
|
||||
File "xmlschema.pxi", line 102, in lxml.etree.XMLSchema.__init__ (src/lxml/lxml.etree.c:154475)
|
||||
lxml.etree.XMLSchemaParseError: local list type: A type, derived by list or union, must have the simple ur-type definition as base type, not '{http://www.opengis.net/gml}doubleList'., line 1
|
||||
|
||||
The XSD validation used by the spatial harvesters requires libxml2 ersion 2.9.
|
||||
|
||||
With CKAN you would probably have installed an older version from your
|
||||
distribution. (e.g. with ``sudo apt-get install libxml2-dev``). You need to
|
||||
find the SO files for the old version::
|
||||
|
||||
$ find /usr -name "libxml2.so"
|
||||
|
||||
For example, it may show it here: ``/usr/lib/x86_64-linux-gnu/libxml2.so``.
|
||||
The directory of the SO file is used as a parameter to the ``configure`` next
|
||||
on.
|
||||
|
||||
Download the libxml2 source::
|
||||
|
||||
$ cd ~
|
||||
$ wget ftp://xmlsoft.org/libxml2/libxml2-2.9.0.tar.gz
|
||||
|
||||
Unzip it::
|
||||
|
||||
$ tar zxvf libxml2-2.9.0.tar.gz
|
||||
$ cd libxml2-2.9.0/
|
||||
|
||||
Configure with the SO directory you found before::
|
||||
|
||||
$ ./configure --libdir=/usr/lib/x86_64-linux-gnu
|
||||
|
||||
Now make it and install it::
|
||||
|
||||
$ make
|
||||
$ sudo make install
|
||||
|
||||
Now check the install by running xmllint::
|
||||
|
||||
$ xmllint --version
|
||||
xmllint: using libxml version 20900
|
||||
compiled with: Threads Tree Output Push Reader Patterns Writer SAXv1 FTP HTTP DTDValid HTML Legacy C14N Catalog XPath XPointer XInclude Iconv ISO8859X Unicode Regexps Automata Expr Schemas Schematron Modules Debug Zlib
|
||||
|
||||
.. _PostGIS: http://postgis.org
|
||||
.. _ckanext-harvest: https://github.com/okfn/ckanext-harvest
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
==========================
|
||||
Setting up a PostGIS table
|
||||
==========================
|
||||
|
||||
.. note:: The extension will generally set up the table automatically for you,
|
||||
and also running the ``initdb`` command will have the same effect. This
|
||||
section just describes what's going on for those who want to know more.
|
||||
|
||||
To be able to store geometries and perform spatial operations, PostGIS_
|
||||
needs to work with geometry fields. Geometry fields should always be
|
||||
added via the ``AddGeometryColumn`` function::
|
||||
|
||||
CREATE TABLE package_extent(
|
||||
package_id text PRIMARY KEY
|
||||
);
|
||||
|
||||
ALTER TABLE package_extent OWNER TO ckan_default;
|
||||
|
||||
SELECT AddGeometryColumn('package_extent','the_geom', 4326, 'GEOMETRY', 2);
|
||||
|
||||
This will add a geometry column in the ``package_extent`` table called
|
||||
``the_geom``, with the spatial reference system EPSG:4326. The stored
|
||||
geometries will be polygons, with 2 dimensions (The CKAN table uses the
|
||||
GEOMETRY type to support multiple geometry types).
|
||||
|
||||
Have a look a the table definition, and see how PostGIS has created
|
||||
some constraints to ensure that the geometries follow the parameters
|
||||
defined in the geometry column creation::
|
||||
|
||||
# \d package_extent
|
||||
|
||||
Table "public.package_extent"
|
||||
Column | Type | Modifiers
|
||||
------------+----------+-----------
|
||||
package_id | text | not null
|
||||
the_geom | geometry |
|
||||
Indexes:
|
||||
"package_extent_pkey" PRIMARY KEY, btree (package_id)
|
||||
Check constraints:
|
||||
"enforce_dims_the_geom" CHECK (st_ndims(the_geom) = 2)
|
||||
"enforce_srid_the_geom" CHECK (st_srid(the_geom) = 4326)
|
||||
|
||||
.. _PostGIS: http://postgis.org
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
============================
|
||||
Previews for Spatial Formats
|
||||
============================
|
||||
|
||||
The extension includes some plugins that implement the IResourcePreview_
|
||||
interface, allowing to preview spatial resource files. They are based in
|
||||
popular Javascript mapping libraries and should be really easy to extend and
|
||||
adapt to your own needs.
|
||||
|
||||
|
||||
GeoJSON Preview
|
||||
---------------
|
||||
|
||||
.. image:: _static/preview-geojson.png
|
||||
|
||||
The GeoJSON previewer is based on Leaflet_. It will render GeoJSON_ files on a
|
||||
map and add a popup showing the features properties, for those resources that
|
||||
have a format of ``geojson`` or ``gjosn``.
|
||||
|
||||
To enable the GeoJSON previewer you need to add the ``geojson_preview`` plugin
|
||||
to your ini file. This plugin also requires the `resource_proxy`_
|
||||
plugin (Make sure you load the ``resource_proxy`` plugin before any other
|
||||
from the spatial extension)::
|
||||
|
||||
ckan.plugins = resource_proxy geojson_preview
|
||||
|
||||
|
||||
WMS Preview
|
||||
-----------
|
||||
|
||||
.. image:: _static/preview-wms.png
|
||||
|
||||
The WMS previewer is based o OpenLayers_. When the plugin is enabled, if
|
||||
datasets contain a resource that has ``wms`` format, the resource page will
|
||||
load a simple map viewer that will attempt to load the remote service layers,
|
||||
based on the GetCapabilities response.
|
||||
|
||||
To enable the WMS previewer you need to add the ``wms_preview`` plugin to your
|
||||
ini file. This plugin also requires the `resource_proxy`_
|
||||
plugin (Make sure you load the ``resource_proxy`` plugin before any other
|
||||
from the spatial extension::
|
||||
|
||||
ckan.plugins = resource_proxy wms_preview
|
||||
|
||||
.. note:: Please note that the WMS previewer included in ckanext-spatial is
|
||||
just a proof of concept and has important limitations, and is
|
||||
just intended as a bootstrap for developers willing to build a more
|
||||
sophisticated one.
|
||||
|
||||
Some projects that have built more advanced map viewers and
|
||||
integrated them with CKAN include:
|
||||
|
||||
* Data.gov.uk (http://data.gov.uk):
|
||||
- https://github.com/datagovuk/ckanext-dgu
|
||||
- https://github.com/datagovuk/ckanext-os
|
||||
|
||||
* Catalog.data.gov (http://catalog.data.gov):
|
||||
- https://github.com/okfn/ckanext-geodatagov
|
||||
- https://github.com/chilukey/viewer
|
||||
|
||||
|
||||
|
||||
.. _IResourcePreview: http://docs.ckan.org/en/latest/writing-extensions.html#ckan.plugins.interfaces.IResourcePreview
|
||||
.. _resource_proxy: http://docs.ckan.org/en/latest/data-viewer.html#viewing-remote-resources-the-resource-proxy
|
||||
.. _Leaflet: http://leafletjs.org
|
||||
.. _GeoJSON: http://geojson.org
|
||||
.. _OpenLayers: http://openlayers.org
|
||||
|
||||
|
|
@ -0,0 +1,297 @@
|
|||
==============
|
||||
Spatial Search
|
||||
==============
|
||||
|
||||
The spatial extension allows to index datasets with spatial information so they
|
||||
can be filtered via a spatial query. This includes both via the web interface
|
||||
(see the `Spatial Search Widget`_) or via the `action API`_, e.g.::
|
||||
|
||||
POST http://localhost:5000/api/action/package_search
|
||||
{ "q": "Pollution",
|
||||
"facet": "true",
|
||||
"facet.field": "country",
|
||||
"extras": {
|
||||
"ext_bbox": "-7.535093,49.208494,3.890688,57.372349" }
|
||||
}
|
||||
|
||||
.. versionchanged:: 2.0.1
|
||||
Starting from this version the spatial filter it is also supported on GET
|
||||
requests:
|
||||
|
||||
http://localhost:5000/api/action/package_search?q=Pollution&ext_bbox=-7.535093,49.208494,3.890688,57.372349
|
||||
|
||||
|
||||
Setup
|
||||
-----
|
||||
|
||||
To enable the spatial query you need to add the ``spatial_query`` plugin to
|
||||
your ini file. This plugin requires the ``spatial_metadata`` plugin, eg::
|
||||
|
||||
ckan.plugins = [other plugins] spatial_metadata spatial_query
|
||||
|
||||
To define which backend to use for the spatial search use the following
|
||||
configuration option (see `Choosing a backend for the spatial search`_)::
|
||||
|
||||
ckanext.spatial.search_backend = solr
|
||||
|
||||
|
||||
Geo-Indexing your datasets
|
||||
--------------------------
|
||||
|
||||
Regardless of the backend that you are using, in order to make a dataset
|
||||
queryable by location, an special extra must be defined, with its key named
|
||||
'spatial'. The value must be a valid GeoJSON_ geometry, for example::
|
||||
|
||||
{
|
||||
"type":"Polygon",
|
||||
"coordinates":[[[2.05827, 49.8625],[2.05827, 55.7447], [-6.41736, 55.7447], [-6.41736, 49.8625], [2.05827, 49.8625]]]
|
||||
}
|
||||
|
||||
or::
|
||||
|
||||
{
|
||||
"type": "Point",
|
||||
"coordinates": [-3.145,53.078]
|
||||
}
|
||||
|
||||
|
||||
Every time a dataset is created, updated or deleted, the extension will
|
||||
synchronize the information stored in the extra with the geometry table.
|
||||
|
||||
Choosing a backend for the spatial search
|
||||
+++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
There are different backends supported for the spatial search, it is important
|
||||
to understand their differences and the necessary setup required when choosing
|
||||
which one to use.
|
||||
|
||||
The following table summarizes the different spatial search backends:
|
||||
|
||||
+------------------------+---------------+-------------------------------------+-----------------------------------------------------------+-------------------------------------------+
|
||||
| Backend | Solr Versions | Supported geometries | Sorting and relevance | Performance with large number of datasets |
|
||||
+========================+===============+=====================================+===========================================================+===========================================+
|
||||
| ``solr`` | 3.1 to 4.x | Bounding Box | Yes, spatial sorting combined with other query parameters | Good |
|
||||
+------------------------+---------------+-------------------------------------+-----------------------------------------------------------+-------------------------------------------+
|
||||
| ``solr-spatial-field`` | 4.x | Bounding Box, Point and Polygon [1] | Not implemented | Good |
|
||||
+------------------------+---------------+-------------------------------------+-----------------------------------------------------------+-------------------------------------------+
|
||||
| ``postgis`` | 1.3 to 4.x | Bounding Box | Partial, only spatial sorting supported [2] | Poor |
|
||||
+------------------------+---------------+-------------------------------------+-----------------------------------------------------------+-------------------------------------------+
|
||||
|
||||
|
||||
[1] Requires JTS
|
||||
|
||||
[2] Needs ``ckanext.spatial.use_postgis_sorting`` set to True
|
||||
|
||||
|
||||
|
||||
We recommend to use the ``solr`` backend whenever possible. Here are more
|
||||
details about the available options:
|
||||
|
||||
* ``solr`` (Recommended)
|
||||
This option uses normal Solr fields to index the relevant bits of
|
||||
information about the geometry and uses an algorithm function to sort
|
||||
results by relevance, keeping any other non-spatial filtering. It only
|
||||
supports bounding boxes both for the geometries to be indexed and the
|
||||
input query shape. It requires `EDisMax`_ query parser, so it will only
|
||||
work on versions of Solr greater than 3.1 (We recommend using Solr 4.x).
|
||||
|
||||
You will need to add the following fields to your Solr schema file to
|
||||
enable it::
|
||||
|
||||
<fields>
|
||||
<!-- ... -->
|
||||
<field name="bbox_area" type="float" indexed="true" stored="true" />
|
||||
<field name="maxx" type="float" indexed="true" stored="true" />
|
||||
<field name="maxy" type="float" indexed="true" stored="true" />
|
||||
<field name="minx" type="float" indexed="true" stored="true" />
|
||||
<field name="miny" type="float" indexed="true" stored="true" />
|
||||
</fields>
|
||||
|
||||
|
||||
* ``solr-spatial-field``
|
||||
This option uses the `spatial field`_ introduced in Solr 4, which allows
|
||||
to index points, rectangles and more complex geometries (complex geometries
|
||||
will require `JTS`_, check the documentation).
|
||||
Sorting has not yet been implemented, users willing to do so will need to
|
||||
modify the query using the ``before_search`` extension point.
|
||||
|
||||
You will need to add the following field type and field to your Solr
|
||||
schema file to enable it (Check the `Solr documentation`__ for more
|
||||
information on the different parameters, note that you don't need
|
||||
``spatialContextFactory`` if you are not using JTS)::
|
||||
|
||||
<types>
|
||||
<!-- ... -->
|
||||
<fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
|
||||
spatialContextFactory="com.spatial4j.core.context.jts.JtsSpatialContextFactory"
|
||||
distErrPct="0.025"
|
||||
maxDistErr="0.000009"
|
||||
units="degrees" />
|
||||
</types>
|
||||
<fields>
|
||||
<!-- ... -->
|
||||
<field name="spatial_geom" type="location_rpt" indexed="true" stored="true" multiValued="true" />
|
||||
</fields>
|
||||
|
||||
* ``postgis``
|
||||
This is the original implementation of the spatial search. It
|
||||
does not require any change in the Solr schema and can run on Solr 1.x,
|
||||
but it is not as efficient as the previous ones. Basically the bounding
|
||||
box based query is performed in PostGIS first, and the ids of the matched
|
||||
datasets are added as a filter to the Solr request. This, apart from being
|
||||
much less efficient, can led to issues on Solr due to size of the requests
|
||||
(See `Solr configuration issues on legacy PostGIS backend`_). There is
|
||||
support for a spatial ranking on this backend (setting
|
||||
``ckanext.spatial.use_postgis_sorting`` to True on the ini file), but
|
||||
it can not be combined with any other filtering.
|
||||
|
||||
|
||||
Spatial Search Widget
|
||||
---------------------
|
||||
|
||||
|
||||
.. image:: _static/spatial-search-widget.png
|
||||
|
||||
The extension provides a snippet to add a map widget to the search form, which
|
||||
allows filtering results by an area of interest.
|
||||
|
||||
To add the map widget to the to the sidebar of the search page, add this to the
|
||||
dataset search page template
|
||||
(``myproj/ckanext/myproj/templates/package/search.html``)::
|
||||
|
||||
{% block secondary_content %}
|
||||
|
||||
{% snippet "spatial/snippets/spatial_query.html" %}
|
||||
|
||||
{% endblock %}
|
||||
|
||||
By default the map widget will show the whole world. If you want to set up a
|
||||
different default extent, you can pass an extra ``default_extent`` to the
|
||||
snippet, either with a pair of coordinates like this::
|
||||
|
||||
{% snippet "spatial/snippets/spatial_query.html", default_extent="[[15.62,
|
||||
-139.21], [64.92, -61.87]]" %}
|
||||
|
||||
or with a GeoJSON object describing a bounding box (note the escaped quotes)::
|
||||
|
||||
{% snippet "spatial/snippets/spatial_query.html", default_extent="{ \"type\":
|
||||
\"Polygon\", \"coordinates\": [[[74.89, 29.39],[74.89, 38.45], [60.50,
|
||||
38.45], [60.50, 29.39], [74.89, 29.39]]]}" %}
|
||||
|
||||
You need to load the ``spatial_metadata`` and ``spatial_query`` plugins to use this
|
||||
snippet.
|
||||
|
||||
|
||||
|
||||
Dataset Extent Map
|
||||
------------------
|
||||
|
||||
.. image:: _static/dataset-extent-map.png
|
||||
|
||||
Using the snippets provided, if datasets contain a ``spatial`` extra like the
|
||||
one described in the previous section, a map will be shown on the dataset
|
||||
details page.
|
||||
|
||||
There are snippets already created to load the map on the left sidebar or in
|
||||
the main body of the dataset details page, but these can be easily modified to
|
||||
suit your project needs
|
||||
|
||||
To add a map to the sidebar, add this to the dataset details page template (eg
|
||||
``myproj/ckanext/myproj/templates/package/read.html``)::
|
||||
|
||||
{% block secondary_content %}
|
||||
{{ super() }}
|
||||
|
||||
{% set dataset_extent = h.get_pkg_dict_extra(c.pkg_dict, 'spatial', '') %}
|
||||
{% if dataset_extent %}
|
||||
{% snippet "spatial/snippets/dataset_map_sidebar.html", extent=dataset_extent %}
|
||||
{% endif %}
|
||||
|
||||
{% endblock %}
|
||||
|
||||
For adding the map to the main body, add this::
|
||||
|
||||
{% block primary_content %}
|
||||
|
||||
<!-- ... -->
|
||||
|
||||
<article class="module prose">
|
||||
|
||||
<!-- ... -->
|
||||
|
||||
{% set dataset_extent = h.get_pkg_dict_extra(c.pkg_dict, 'spatial', '') %}
|
||||
{% if dataset_extent %}
|
||||
{% snippet "spatial/snippets/dataset_map.html", extent=dataset_extent %}
|
||||
{% endif %}
|
||||
|
||||
</article>
|
||||
{% endblock %}
|
||||
|
||||
|
||||
You need to load the ``spatial_metadata`` plugin to use these snippets.
|
||||
|
||||
Legacy Search
|
||||
-------------
|
||||
|
||||
Solr configuration issues on legacy PostGIS backend
|
||||
+++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
.. warning::
|
||||
|
||||
If you find any of the issues described in this section it is strongly
|
||||
recommended that you consider switching to one of the Solr based backends
|
||||
which are much more efficient. These notes are just kept for informative
|
||||
purposes.
|
||||
|
||||
|
||||
If using Spatial Query functionality then there is an additional SOLR/Lucene
|
||||
setting that should be used to set the limit on number of datasets searchable
|
||||
with a spatial value.
|
||||
|
||||
The setting is ``maxBooleanClauses`` in the solrconfig.xml and the value is the
|
||||
number of datasets spatially searchable. The default is ``1024`` and this could
|
||||
be increased to say ``16384``. For a SOLR single core this will probably be at
|
||||
`/etc/solr/conf/solrconfig.xml`. For a multiple core set-up, there will me
|
||||
several solrconfig.xml files a couple of levels below `/etc/solr`. For that
|
||||
case, *all* of the cores' `solrconfig.xml` should have this setting at the new
|
||||
value.
|
||||
|
||||
Example::
|
||||
|
||||
<maxBooleanClauses>16384</maxBooleanClauses>
|
||||
|
||||
This setting is needed because PostGIS spatial query results are fed into SOLR
|
||||
using a Boolean expression, and the parser for that has a limit. So if your
|
||||
spatial area contains more than the limit (of which the default is 1024) then
|
||||
you will get this error::
|
||||
|
||||
Dataset search error: ('SOLR returned an error running query...
|
||||
|
||||
and in the SOLR logs you see::
|
||||
|
||||
too many boolean clauses ... Caused by:
|
||||
org.apache.lucene.search.BooleanQuery$TooManyClauses: maxClauseCount is set to
|
||||
1024
|
||||
|
||||
|
||||
Legacy API
|
||||
++++++++++
|
||||
|
||||
The extension adds the following call to the CKAN search API, which returns
|
||||
datasets with an extent that intersects with the bounding box provided::
|
||||
|
||||
/api/2/search/dataset/geo?bbox={minx,miny,maxx,maxy}[&crs={srid}]
|
||||
|
||||
If the bounding box coordinates are not in the same projection as the one
|
||||
defined in the database, a CRS must be provided, in one of the following forms:
|
||||
|
||||
- `urn:ogc:def:crs:EPSG::4326`
|
||||
- EPSG:4326
|
||||
- 4326
|
||||
|
||||
.. _action API: http://docs.ckan.org/en/latest/apiv3.html
|
||||
.. _edismax: http://wiki.apache.org/solr/ExtendedDisMax
|
||||
.. _JTS: http://www.vividsolutions.com/jts/JTSHome.htm
|
||||
.. _spatial field: http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4
|
||||
__ `spatial field`_
|
||||
.. _GeoJSON: http://geojson.org
|
||||
Loading…
Reference in New Issue