simple test DAG

2024-03-06 23:31:24 +01:00 · 2024-03-06 23:31:24 +01:00 · 379920e21b
parent 5d073deaa7
commit 379920e21b
1 changed files with 1 additions and 3 deletions
--- a/airflow/dags/populate_opensearch.py
+++ b/airflow/dags/populate_opensearch.py
@ -64,9 +64,7 @@ def bulk_load():
                verify=False)

    for key in keys:
-        key_obj = hook.get_key(key, bucket_name=S3_BUCKET_NAME)
-
-        s3_obj = hook.Object(S3_BUCKET_NAME, key)
+        s3_obj = hook.get_key(key, bucket_name=S3_BUCKET_NAME)
        with gzip.GzipFile(fileobj=s3_obj.get()["Body"]) as gzipfile:
            for line in gzipfile:
                session.post("https://opensearch-cluster.lot1-opensearch-cluster.svc.local/organization_index/_doc",