D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

print existing files

This commit is contained in:
Giambattista Bloisi 2024-10-19 10:15:30 +02:00
parent 448bb924ab
commit ba3f351736
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
workflow/dnet/S3_untar.py
View File

@ -69,8 +69,11 @@ def s3_untar():
dst_bucket = context['params']['dst_bucket']
dst_key_prefix = os.path.normpath(context["params"]["dst_key_prefix"])
print(f"Existing keys with prefix: {dst_key_prefix}/")
existing_keys = dict.fromkeys(hook.list_keys(bucket_name=dst_bucket,
prefix=dst_key_prefix + "/"), 0)
for k in existing_keys.keys():
print(f"{k}")
s3_obj = hook.get_key(context["params"]["src_key"], bucket_name=src_bucket)
with tarfile.open(fileobj=s3_obj.get()["Body"], mode='r|*') as tar:
@ -81,7 +84,7 @@ def s3_untar():
print(f"Skipping {member.name}: is not a file")
continue
if dst_key in existing_keys:
print(f"Skipping {member.name}: already exists")
print(f"Skipping {member.name}: already exists as {dst_key}")
continue
print(f"Extracting {member.name} to {dst_key}")
fileobj = tar.extractfile(member)