2011-03-10 15:02:21 +01:00
|
|
|
from ckan.lib.helpers import json
|
2011-03-10 10:45:16 +01:00
|
|
|
import urllib2
|
2011-03-09 19:56:55 +01:00
|
|
|
import ckan.lib.helpers as h
|
|
|
|
from ckan.lib.base import BaseController, c, g, request, \
|
2011-03-10 15:02:21 +01:00
|
|
|
response, session, render, config, abort, redirect
|
2011-03-09 19:56:55 +01:00
|
|
|
#from ..dictization import *
|
|
|
|
|
|
|
|
class ViewController(BaseController):
|
|
|
|
|
2011-03-10 15:02:21 +01:00
|
|
|
api_url = config.get('ckan.api_url', 'http://localhost:5000').rstrip('/')+'/api/2/rest'
|
2011-03-10 17:48:50 +01:00
|
|
|
form_api_url = config.get('ckan.api_url', 'http://localhost:5000').rstrip('/')+'/api/2/form'
|
2011-03-10 16:32:51 +01:00
|
|
|
api_key = config.get('ckan.harvesting.api_key')
|
2011-03-10 10:45:16 +01:00
|
|
|
|
2011-03-10 16:32:51 +01:00
|
|
|
def _do_request(self,url,data = None):
|
2011-03-10 10:45:16 +01:00
|
|
|
|
2011-03-10 16:32:51 +01:00
|
|
|
http_request = urllib2.Request(
|
|
|
|
url = url,
|
|
|
|
headers = {'Authorization' : self.api_key}
|
|
|
|
)
|
|
|
|
|
|
|
|
if data:
|
|
|
|
http_request.add_data(data)
|
|
|
|
|
|
|
|
try:
|
|
|
|
return urllib2.urlopen(http_request)
|
2011-03-10 15:02:21 +01:00
|
|
|
except urllib2.HTTPError as e:
|
2011-03-11 13:35:27 +01:00
|
|
|
raise
|
2011-03-10 16:32:51 +01:00
|
|
|
|
|
|
|
|
|
|
|
def index(self):
|
|
|
|
# Request all harvesting sources
|
|
|
|
sources_url = self.api_url + '/harvestsource'
|
|
|
|
|
|
|
|
doc = self._do_request(sources_url).read()
|
|
|
|
sources_ids = json.loads(doc)
|
|
|
|
|
|
|
|
source_url = sources_url + '/%s'
|
|
|
|
sources = []
|
|
|
|
|
|
|
|
# For each source, request its details
|
|
|
|
for source_id in sources_ids:
|
|
|
|
doc = self._do_request(source_url % source_id).read()
|
|
|
|
sources.append(json.loads(doc))
|
|
|
|
|
|
|
|
c.sources = sources
|
|
|
|
return render('ckanext/harvest/index.html')
|
2011-03-10 15:02:21 +01:00
|
|
|
|
2011-03-09 19:56:55 +01:00
|
|
|
def create(self):
|
2011-03-10 15:02:21 +01:00
|
|
|
|
2011-03-09 19:56:55 +01:00
|
|
|
# This is the DGU form API, so we don't use self.api_url
|
2011-03-10 17:48:50 +01:00
|
|
|
form_url = self.form_api_url + '/harvestsource/create'
|
2011-03-09 19:56:55 +01:00
|
|
|
if request.method == 'GET':
|
2011-03-10 15:02:21 +01:00
|
|
|
# Request the fields
|
2011-03-10 16:32:51 +01:00
|
|
|
c.form = self._do_request(form_url).read()
|
2011-03-10 15:02:21 +01:00
|
|
|
|
2011-03-09 19:56:55 +01:00
|
|
|
return render('ckanext/harvest/create.html')
|
|
|
|
if request.method == 'POST':
|
2011-03-10 15:02:21 +01:00
|
|
|
# Build an object like the one expected by the DGU form API
|
|
|
|
data = {
|
|
|
|
'form_data':
|
|
|
|
{'HarvestSource--url': request.POST['HarvestSource--url'],
|
|
|
|
'HarvestSource--description': request.POST['HarvestSource--description']},
|
|
|
|
'user_ref':'',
|
|
|
|
'publisher_ref':''
|
|
|
|
}
|
|
|
|
data = json.dumps(data)
|
2011-03-11 13:35:27 +01:00
|
|
|
try:
|
|
|
|
r = self._do_request(form_url,data)
|
|
|
|
|
|
|
|
h.flash_success('Harvesting source added successfully')
|
|
|
|
except urllib2.HTTPError as e:
|
|
|
|
msg = 'An error occurred: [%s %s]' % (str(e.getcode()),e.msg)
|
|
|
|
# The form API returns just a 500, so we are not exactly sure of what
|
|
|
|
# happened, but most probably it was a duplicate entry
|
|
|
|
msg = msg + ' Does the source already exist?'
|
|
|
|
h.flash_error(msg)
|
|
|
|
finally:
|
|
|
|
redirect(h.url_for(controller='harvest', action='index'))
|
|
|
|
|
|
|
|
def show(self,id):
|
|
|
|
sources_url = self.api_url + '/harvestsource/%s' % id
|
|
|
|
doc = self._do_request(sources_url).read()
|
|
|
|
c.source = json.loads(doc)
|
|
|
|
|
2011-03-10 18:24:23 +01:00
|
|
|
def create_harvesting_job(self,id):
|
2011-03-10 17:48:50 +01:00
|
|
|
form_url = self.api_url + '/harvestingjob'
|
|
|
|
data = {
|
2011-03-10 18:24:23 +01:00
|
|
|
'source_id': id,
|
2011-03-10 17:48:50 +01:00
|
|
|
'user_ref': ''
|
|
|
|
}
|
|
|
|
data = json.dumps(data)
|
2011-03-11 13:35:27 +01:00
|
|
|
try:
|
|
|
|
r = self._do_request(form_url,data)
|
2011-03-09 19:56:55 +01:00
|
|
|
|
2011-03-11 13:35:27 +01:00
|
|
|
h.flash_success('Refresh requested, harvesting will take place within 15 minutes.')
|
|
|
|
except urllib2.HTTPError as e:
|
|
|
|
msg = 'An error occurred: [%s %s]' % (str(e.getcode()),e.msg)
|
|
|
|
if e.getcode() == 400:
|
|
|
|
msg = msg + ' ' + e.read()
|
|
|
|
|
|
|
|
h.flash_error(msg)
|
|
|
|
finally:
|
|
|
|
redirect(h.url_for(controller='harvest', action='index', id=None))
|
2011-03-09 19:56:55 +01:00
|
|
|
return render('ckanext/harvest/show.html')
|