giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

simple test DAG

This commit is contained in:
Giambattista Bloisi 2024-03-06 23:31:24 +01:00
parent 5d073deaa7
commit 379920e21b
1 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
airflow/dags/populate_opensearch.py
View File

@ -64,9 +64,7 @@ def bulk_load():
verify=False) verify=False)
for key in keys: for key in keys:
key_obj = hook.get_key(key, bucket_name=S3_BUCKET_NAME) s3_obj = hook.get_key(key, bucket_name=S3_BUCKET_NAME)
s3_obj = hook.Object(S3_BUCKET_NAME, key)
with gzip.GzipFile(fileobj=s3_obj.get()["Body"]) as gzipfile: with gzip.GzipFile(fileobj=s3_obj.get()["Body"]) as gzipfile:
for line in gzipfile: for line in gzipfile:
session.post("https://opensearch-cluster.lot1-opensearch-cluster.svc.local/organization_index/_doc", session.post("https://opensearch-cluster.lot1-opensearch-cluster.svc.local/organization_index/_doc",