giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

simple test DAG

This commit is contained in:
Giambattista Bloisi 2024-03-21 10:49:44 +01:00
parent d660233e8e
commit 83b86b50ab
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
airflow/dags/import_skg_if.py
View File

@ -131,7 +131,7 @@ def skg_if_pipeline():
pieces = [] pieces = []
for entity in ENTITIES: for entity in ENTITIES:
keys = hook.list_keys(bucket_name=S3_BUCKET_NAME, prefix=f'{entity}/') keys = hook.list_keys(bucket_name=S3_BUCKET_NAME, prefix=f'{entity}/')
to_delete = filter(lambda key: key.endswith('.PROCESSED'), keys) to_delete = list(filter(lambda key: key.endswith('.PROCESSED'), keys))
hook.delete_objects(bucket=S3_BUCKET_NAME,keys=to_delete) hook.delete_objects(bucket=S3_BUCKET_NAME,keys=to_delete)
for key in keys: for key in keys:
if key.endswith('.gz'): if key.endswith('.gz'):