simple test DAG

airflow/dags/populate_opensearch.py

@@ -18,6 +18,7 @@
 """Example HTTP operator and sensor"""
 from __future__ import annotations
 
+import io
 import json
 import gzip
 import json
@@ -74,7 +75,8 @@ def bulk_load():
         print(key)
         s3_obj = hook.get_key(key, bucket_name=S3_BUCKET_NAME)
         with gzip.GzipFile(fileobj=s3_obj.get()["Body"]) as gzipfile:
-            for line in gzipfile:
+            buff = io.BufferedReader(gzipfile)
+            for line in buff:
                 data = json.loads(line)
                 session.post("https://opensearch-cluster.lot1-opensearch-cluster.svc.cluster.local:9200/organization_index/_doc/" + requests.utils.quote(data['id'], safe='') + "?refresh=false",
                              data=json.dumps(data),