move detecting unchanged datasets to ckanharvester and queue.py

ckanext/harvest/harvesters/base.py

@@ -178,8 +178,6 @@ class HarvesterBase(SingletonPlugin):
 
                 else:
                     log.info('Package with GUID %s not updated, skipping...' % harvest_object.guid)
-                    harvest_object.report_status = 'not modified'
-                    harvest_object.save()
                     return
 
                 # Flag the other objects linking to this package as not current anymore

ckanext/harvest/harvesters/ckanharvester.py

@@ -378,6 +378,17 @@ class CKANHarvester(HarvesterBase):
             for resource in package_dict.get('resources', []):
                 resource.pop('url_type', None)
 
+            # Check if package exists
+            data_dict = {}
+            data_dict['id'] = package_dict['id']
+            try:
+                existing_package_dict = get_action('package_show')(context, data_dict)
+                if 'metadata_modified' in package_dict and \
+                                package_dict['metadata_modified'] <= existing_package_dict.get('metadata_modified'):
+                    return "unchanged"
+            except NotFound:
+                pass
+
             result = self._create_or_update_package(package_dict,harvest_object)
 
             if result and self.config.get('read_only',False) == True:

ckanext/harvest/queue.py

@@ -312,14 +312,16 @@ def fetch_and_import_stages(harvester, obj):
         obj.import_finished = datetime.datetime.utcnow()
         if success_import:
             obj.state = "COMPLETE"
-        else:
-            obj.state = "ERROR"
+            if success_import is 'unchanged':
+                obj.report_status = 'not modified'
                 obj.save()
-    else:
-        obj.state = "ERROR"
-        obj.save()
-    if obj.report_status == 'not modified':
                 return
+        else:
+            obj.state = "ERROR"
+        obj.save()
+    else:
+        obj.state = "ERROR"
+        obj.save()
     if obj.state == 'ERROR':
         obj.report_status = 'errored'
     elif obj.current == False: