giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-07-03 00:59:53 +02:00
parent 0339a92de5
commit 52dd788d15
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
airflow/dags/S3_untar.py
View File

@ -41,8 +41,10 @@ def s3_untar():
dst_key = os.path.normpath(dst_key) dst_key = os.path.normpath(dst_key)
# Ignore directories, links, devices, fifos, etc. # Ignore directories, links, devices, fifos, etc.
if (not member.isfile()) or member.name.endswith('/'): if (not member.isfile()) or member.name.endswith('/'):
print(f"Skipping {member.name}") print(f"Skipping {member.name}: is not a file")
continue continue
if hook.check_for_key(key=dst_key, bucket_name=context["params"]["dst_bucket"]):
print(f"Skipping {member.name}: already exists")
print(f"Extracting {member.name} to {dst_key}") print(f"Extracting {member.name} to {dst_key}")
fileobj = tar.extractfile(member) fileobj = tar.extractfile(member)
fileobj.seekable = lambda: False fileobj.seekable = lambda: False