giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

simple test DAG

This commit is contained in:
Giambattista Bloisi 2024-03-06 23:42:25 +01:00
parent d6bfc955a3
commit 2c81ded53c
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
airflow/dags/populate_opensearch.py
View File

@ -54,7 +54,7 @@ def bulk_load():
session = requests.Session()
session.auth = ("admin", "admin")
session.put("https://opensearch-cluster.lot1-opensearch-cluster.svc.local/organization_index",
session.put("https://opensearch-cluster.lot1-opensearch-cluster.svc.cluster.local/organization_index",
data=json.dumps({
"settings": {
"number_of_shards": 1,
@ -68,7 +68,7 @@ def bulk_load():
s3_obj = hook.get_key(key, bucket_name=S3_BUCKET_NAME)
with gzip.GzipFile(fileobj=s3_obj.get()["Body"]) as gzipfile:
for line in gzipfile:
session.post("https://opensearch-cluster.lot1-opensearch-cluster.svc.local/organization_index/_doc",
session.post("https://opensearch-cluster.lot1-opensearch-cluster.svc.cluster.local/organization_index/_doc",
data=line,
headers={"Content-Type": "application/json"},
verify=False)
@ -80,6 +80,6 @@ with DAG(
start_date=datetime(2021, 1, 1),
catchup=False,
) as dag:
bulk_load_data = bulk_load()
bulk_load_data = bulk_load("")
chain(bulk_load_data)