Merge branch '91-reindex-list-fields-in-config'

ckanext/harvest/logic/action/update.py

@@ -1,4 +1,5 @@
 import hashlib
+import json
 
 import logging
 import datetime
@@ -133,13 +134,7 @@ def harvest_source_clear(context,data_dict):
     model.Session.execute(sql)
 
     # Refresh the index for this source to update the status object
-    context.update({'validate': False, 'ignore_auth': True})
+    get_action('harvest_source_reindex')(context, {'id': harvest_source_id})
-    package_dict = logic.get_action('package_show')(context,
-            {'id': harvest_source_id})
-
-    if package_dict:
-        package_index = PackageSearchIndex()
-        package_index.index_package(package_dict)
 
     return {'id': harvest_source_id}
 
@@ -326,15 +321,9 @@ def harvest_jobs_run(context,data_dict):
                     job_obj.save()
                     # Reindex the harvest source dataset so it has the latest
                     # status
-                    if 'extras_as_string'in context:
+                    get_action('harvest_source_reindex')(reindex_context,
-                        del context['extras_as_string']
-                    context.update({'validate': False, 'ignore_auth': True})
-                    package_dict = logic.get_action('package_show')(context,
                         {'id': job_obj.source.id})
 
-                    if package_dict:
-                        package_index.index_package(package_dict)
-
     # resubmit old redis tasks
     resubmit_jobs()
 
@@ -361,6 +350,8 @@ def harvest_jobs_run(context,data_dict):
     publisher.close()
     return sent_jobs
 
+
+@logic.side_effect_free
 def harvest_sources_reindex(context, data_dict):
     '''
         Reindexes all harvest source datasets with the latest status
@@ -376,14 +367,37 @@ def harvest_sources_reindex(context, data_dict):
                             .all()
 
     package_index = PackageSearchIndex()
+
+    reindex_context = {'defer_commit': True}
     for package in packages:
+        get_action('harvest_source_reindex')(reindex_context, {'id': package.id})
+
+    package_index.commit()
+
+    return True
+
+@logic.side_effect_free
+def harvest_source_reindex(context, data_dict):
+    '''Reindex a single harvest source'''
+
+    harvest_source_id = logic.get_or_bust(data_dict, 'id')
+    defer_commit = context.get('defer_commit', False)
+
     if 'extras_as_string'in context:
         del context['extras_as_string']
     context.update({'ignore_auth': True})
     package_dict = logic.get_action('harvest_source_show')(context,
-            {'id': package.id})
+        {'id': harvest_source_id})
-        log.debug('Updating search index for harvest source {0}'.format(package.id))
+    log.debug('Updating search index for harvest source {0}'.format(harvest_source_id))
-        package_index.index_package(package_dict, defer_commit=True)
 
-    package_index.commit()
+    # Remove configuration values
-    log.info('Updated search index for {0} harvest sources'.format(len(packages)))
+    new_dict = {}
+    if package_dict.get('config'):
+        config = json.loads(package_dict['config'])
+        for key, value in package_dict.iteritems():
+            if key not in config:
+                new_dict[key] = value
+    package_index = PackageSearchIndex()
+    package_index.index_package(new_dict, defer_commit=defer_commit)
+
+    return True

ckanext/harvest/logic/auth/update.py

@@ -68,3 +68,11 @@ def harvest_sources_reindex(context, data_dict):
         return {'success': False, 'msg': pt._('Only sysadmins can reindex all harvest sources')}
     else:
         return {'success': True}
+
+def harvest_source_reindex(context, data_dict):
+    '''
+        Authorization check for reindexing a harvest source
+
+        It forwards to harvest_source_update
+    '''
+    return harvest_source_update(context, data_dict)