CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[lib] Ignore deleted packages in source stats

This commit is contained in:
amercader 2012-01-25 17:47:35 +00:00
parent 3a489bbb82
commit a997e45470
1 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
ckanext/harvest/lib/__init__.py
View File

@ -47,7 +47,7 @@ def _get_source_status(source, detailed=True):
out['last_harvest_request'] = str(last_job.gather_finished) out['last_harvest_request'] = str(last_job.gather_finished)
#Get HarvestObjects from last job whit links to packages #Get HarvestObjects from last job whit links to packages
if detailed: if detailed:
last_objects = [obj for obj in last_job.objects if obj.package is not None] last_objects = [obj for obj in last_job.objects if obj.package is not None]
if len(last_objects) == 0: if len(last_objects) == 0:
@ -77,7 +77,7 @@ def _get_source_status(source, detailed=True):
out['last_harvest_statistics']['errors'] = len(last_job.gather_errors) \ out['last_harvest_statistics']['errors'] = len(last_job.gather_errors) \
+ object_errors.count() + object_errors.count()
if detailed: if detailed:
for gather_error in last_job.gather_errors: for gather_error in last_job.gather_errors:
out['last_harvest_errors']['gather'].append(gather_error.message) out['last_harvest_errors']['gather'].append(gather_error.message)
@ -88,7 +88,8 @@ def _get_source_status(source, detailed=True):
# Overall statistics # Overall statistics
packages = Session.query(distinct(HarvestObject.package_id),Package.name) \ packages = Session.query(distinct(HarvestObject.package_id),Package.name) \
.join(Package).join(HarvestJob).join(HarvestSource) \ .join(Package).join(HarvestJob).join(HarvestSource) \
.filter(HarvestJob.source==source) .filter(HarvestJob.source==source) \
.filter(Package.state==u'active')
out['overall_statistics']['added'] = packages.count() out['overall_statistics']['added'] = packages.count()
if detailed: if detailed:
@ -348,17 +349,23 @@ def import_last_objects(source_id=None):
if not source: if not source:
raise NotFound('Harvest source %s does not exist' % source_id) raise NotFound('Harvest source %s does not exist' % source_id)
if not source.active:
raise Exception('This harvest source is not active')
last_objects_ids = Session.query(HarvestObject.id) \ last_objects_ids = Session.query(HarvestObject.id) \
.join(HarvestJob) \ .join(HarvestJob).join(Package) \
.filter(HarvestJob.source==source) \ .filter(HarvestJob.source==source) \
.filter(HarvestObject.package!=None) \ .filter(HarvestObject.package!=None) \
.filter(Package.state==u'active') \
.order_by(HarvestObject.guid) \ .order_by(HarvestObject.guid) \
.order_by(HarvestObject.metadata_modified_date.desc()) \ .order_by(HarvestObject.metadata_modified_date.desc()) \
.order_by(HarvestObject.gathered.desc()) \ .order_by(HarvestObject.gathered.desc()) \
.all() .all()
else: else:
last_objects_ids = Session.query(HarvestObject.id) \ last_objects_ids = Session.query(HarvestObject.id) \
.join(Package) \
.filter(HarvestObject.package!=None) \ .filter(HarvestObject.package!=None) \
.filter(Package.state==u'active') \
.order_by(HarvestObject.guid) \ .order_by(HarvestObject.guid) \
.order_by(HarvestObject.metadata_modified_date.desc()) \ .order_by(HarvestObject.metadata_modified_date.desc()) \
.order_by(HarvestObject.gathered.desc()) \ .order_by(HarvestObject.gathered.desc()) \
@ -382,7 +389,7 @@ def import_last_objects(source_id=None):
return imported_objects return imported_objects
def create_harvest_job_all(): def create_harvest_job_all():
# Get all active sources # Get all active sources
sources = get_harvest_sources(active=True) sources = get_harvest_sources(active=True)
jobs = [] jobs = []