giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

simple test DAG

This commit is contained in:
Giambattista Bloisi 2024-03-08 16:51:39 +01:00
parent deb6567a73
commit c259c529bc
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
airflow/dags/skg_if_pipeline.py
View File

@ -51,6 +51,7 @@ def unzip_to_s3(key: str, bucket: str):
continue continue
local_file_path = os.path.join(root, file) local_file_path = os.path.join(root, file)
hook.load_file(local_file_path, strip_prefix(local_file_path, tmp_dir), S3_BUCKET_NAME, replace=True) hook.load_file(local_file_path, strip_prefix(local_file_path, tmp_dir), S3_BUCKET_NAME, replace=True)
return ""
@task @task
@ -88,6 +89,7 @@ def bulk_load(entity: str):
verify=False) verify=False)
session.post(f'https://opensearch-cluster.lot1-opensearch-cluster.svc.cluster.local:9200/{entity}_index/_refresh', session.post(f'https://opensearch-cluster.lot1-opensearch-cluster.svc.cluster.local:9200/{entity}_index/_refresh',
verify=False) verify=False)
return ""
with DAG( with DAG(