initial stage

2024-03-24 19:04:57 +01:00 · 2024-03-24 19:04:57 +01:00 · 921ce0bf48
parent 99ef9b3980
commit 921ce0bf48
1 changed files with 2 additions and 1 deletions
--- a/airflow/dags/import_EOSC_graph.py
+++ b/airflow/dags/import_EOSC_graph.py
@ -132,7 +132,8 @@ def import_EOSC_graph():
        for entity in ENTITIES:
            keys = hook.list_keys(bucket_name=S3_BUCKET_NAME, prefix=f'{entity}/')
            to_delete = list(filter(lambda key: key.endswith('.PROCESSED'), keys))
-            hook.delete_objects(bucket=S3_BUCKET_NAME,keys=to_delete)
+            if len(to_delete) > 0:
+                hook.delete_objects(bucket=S3_BUCKET_NAME,keys=to_delete)
            for key in keys:
                if key.endswith('.gz'):
                    pieces.append((entity, key))