From 47bcc93c7dfd3d63511247b612ee537839f5f078 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Thu, 13 Jun 2024 00:11:39 +0200 Subject: [PATCH] initial stage --- airflow/dags/antispam-batch.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/airflow/dags/antispam-batch.py b/airflow/dags/antispam-batch.py index d68e4c7..8c690bd 100644 --- a/airflow/dags/antispam-batch.py +++ b/airflow/dags/antispam-batch.py @@ -30,13 +30,13 @@ dag = DAG( secrets = [ Secret( deploy_type='env', - deploy_target='CURATION_OPENSEARCH_USER', + deploy_target='CURATION_OPENSEARCH__USER', secret='opensearch-conn-secrets', key='username', ), Secret( deploy_type='env', - deploy_target='CURATION_OPENSEARCH_PASSWORD', + deploy_target='CURATION_OPENSEARCH__PASSWORD', secret='opensearch-conn-secrets', key='password', ), @@ -49,16 +49,14 @@ task = KubernetesPodOperator( namespace='kg-airflow', image='gbloisi/curation:1.0.0', image_pull_policy="Always", - cmds=['sleep'], - arguments=['9999999'], - # cmds=['python3'], - # arguments=['/antispam-batch.py', - # "--opensearch.host", conn.host, - # "--opensearch.port", str(conn.port), - # "--openai.host", "local-ai.kg-airflow.dev-1.eosc.intranet", - # "--openai.port", "8000", - # "--parallelism", "8" - # ], + cmds=['python3'], + arguments=['/antispam-batch.py', + "--opensearch.host", conn.host, + "--opensearch.port", str(conn.port), + "--openai.host", "local-ai.kg-airflow.dev-1.eosc.intranet", + "--openai.port", "8000", + "--parallelism", "8" + ], secrets=secrets, is_delete_operator_pod=True, in_cluster=True,