giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-06-10 13:43:52 +02:00
parent 26e8789d30
commit 2e72b11447
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
airflow/dags/S3_untar.py
View File

@ -35,7 +35,7 @@ def s3_untar():
def untar(**context): def untar(**context):
hook = S3Hook(S3_CONN_ID, transfer_config_args={'use_threads': False}) hook = S3Hook(S3_CONN_ID, transfer_config_args={'use_threads': False})
s3_obj = hook.get_key(context["params"]["src_key"], bucket_name=context["params"]["src_bucket"]) s3_obj = hook.get_key(context["params"]["src_key"], bucket_name=context["params"]["src_bucket"])
with tarfile.open(fileobj=s3_obj.get()["Body"], mode='r|') as tar: with tarfile.open(fileobj=s3_obj.get()["Body"], mode='r:*') as tar:
for member in tar: for member in tar:
dst_key = context["params"]["dst_key_prefix"] + "/" + member.name dst_key = context["params"]["dst_key_prefix"] + "/" + member.name
dst_key = os.path.normpath(dst_key) dst_key = os.path.normpath(dst_key)