diff --git a/airflow/dags/import_EOSC_graph.py b/airflow/dags/import_EOSC_graph.py index 58f91d5..9fe5b95 100644 --- a/airflow/dags/import_EOSC_graph.py +++ b/airflow/dags/import_EOSC_graph.py @@ -132,8 +132,8 @@ def import_EOSC_graph(): for entity in ENTITIES: keys = hook.list_keys(bucket_name=S3_BUCKET_NAME, prefix=f'{entity}/') to_delete = list(filter(lambda key: key.endswith('.PROCESSED'), keys)) - if len(to_delete) > 0: - hook.delete_objects(bucket=S3_BUCKET_NAME,keys=to_delete) + for obj in to_delete: + hook.delete_objects(bucket=S3_BUCKET_NAME,keys=obj) for key in keys: if key.endswith('.gz'): pieces.append((entity, key))