giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-03-24 19:07:12 +01:00
parent 921ce0bf48
commit 6b2ef00c25
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
airflow/dags/import_EOSC_graph.py
View File

@ -132,8 +132,8 @@ def import_EOSC_graph():
for entity in ENTITIES: for entity in ENTITIES:
keys = hook.list_keys(bucket_name=S3_BUCKET_NAME, prefix=f'{entity}/') keys = hook.list_keys(bucket_name=S3_BUCKET_NAME, prefix=f'{entity}/')
to_delete = list(filter(lambda key: key.endswith('.PROCESSED'), keys)) to_delete = list(filter(lambda key: key.endswith('.PROCESSED'), keys))
if len(to_delete) > 0: for obj in to_delete:
hook.delete_objects(bucket=S3_BUCKET_NAME,keys=to_delete) hook.delete_objects(bucket=S3_BUCKET_NAME,keys=obj)
for key in keys: for key in keys:
if key.endswith('.gz'): if key.endswith('.gz'):
pieces.append((entity, key)) pieces.append((entity, key))