initial stage

airflow/dags/S3_untar.py

@@ -39,11 +39,15 @@ def s3_untar():
             for member in tar:
                 dst_key = context["params"]["dst_key_prefix"] + member.name
                 dst_key = dst_key.replace('//', '/')
+                # Ignore directories, links, devices, fifos, etc.
+                if not member.isfile():
+                    print(f"Skipping {member.name} to {dst_key}")
+                    continue
                 print(f"Extracting {member.name} to {dst_key}")
-                hook.load_file_obj(tar.extractfile(member),
-                                   dst_key,
-                                   context["params"]["dst_bucket"],
-                                   replace=True)
+                #hook.load_file_obj(tar.extractfile(member),
+                #                   dst_key,
+                #                   context["params"]["dst_bucket"],
+                #                   replace=True)
 
     untar()