From fe50bf1475b7d3a8890e55ced1326cbf6541d7ff Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Mon, 10 Jun 2024 09:28:36 +0200 Subject: [PATCH] initial stage --- airflow/dags/S3_untar.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/airflow/dags/S3_untar.py b/airflow/dags/S3_untar.py index 5349589..6517124 100644 --- a/airflow/dags/S3_untar.py +++ b/airflow/dags/S3_untar.py @@ -39,11 +39,15 @@ def s3_untar(): for member in tar: dst_key = context["params"]["dst_key_prefix"] + member.name dst_key = dst_key.replace('//', '/') + # Ignore directories, links, devices, fifos, etc. + if not member.isfile(): + print(f"Skipping {member.name} to {dst_key}") + continue print(f"Extracting {member.name} to {dst_key}") - hook.load_file_obj(tar.extractfile(member), - dst_key, - context["params"]["dst_bucket"], - replace=True) + #hook.load_file_obj(tar.extractfile(member), + # dst_key, + # context["params"]["dst_bucket"], + # replace=True) untar()