giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-06-10 09:23:18 +02:00
parent 5fceeb8b61
commit d7e3e7a1b7
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
airflow/dags/S3_untar.py
View File

@ -37,8 +37,11 @@ def s3_untar():
s3_obj = hook.get_key(context["params"]["src_key"], bucket_name=context["params"]["src_bucket"]) s3_obj = hook.get_key(context["params"]["src_key"], bucket_name=context["params"]["src_bucket"])
with tarfile.open(fileobj=s3_obj.get()["Body"], mode='r|') as tar: with tarfile.open(fileobj=s3_obj.get()["Body"], mode='r|') as tar:
for member in tar: for member in tar:
hook.load_file_obj(tar.extractfile(member.name), dst_key = context["params"]["dst_key_prefix"] + member.name
context["params"]["dst_key_prefix"] + member.name, dst_key = dst_key.replace('//', '/')
print(f"Extracting {member.name} to {dst_key}")
hook.load_file_obj(tar.extractfile(member),
dst_key,
context["params"]["dst_bucket"], context["params"]["dst_bucket"],
replace=True) replace=True)