giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-06-13 00:11:39 +02:00
parent f807ce4911
commit 47bcc93c7d
1 changed files with 10 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
airflow/dags/antispam-batch.py
View File

@ -30,13 +30,13 @@ dag = DAG(
secrets = [
Secret(
deploy_type='env',
deploy_target='CURATION_OPENSEARCH_USER',
deploy_target='CURATION_OPENSEARCH__USER',
secret='opensearch-conn-secrets',
key='username',
),
Secret(
deploy_type='env',
deploy_target='CURATION_OPENSEARCH_PASSWORD',
deploy_target='CURATION_OPENSEARCH__PASSWORD',
secret='opensearch-conn-secrets',
key='password',
),
@ -49,16 +49,14 @@ task = KubernetesPodOperator(
namespace='kg-airflow',
image='gbloisi/curation:1.0.0',
image_pull_policy="Always",
cmds=['sleep'],
arguments=['9999999'],
# cmds=['python3'],
# arguments=['/antispam-batch.py',
# "--opensearch.host", conn.host,
# "--opensearch.port", str(conn.port),
# "--openai.host", "local-ai.kg-airflow.dev-1.eosc.intranet",
# "--openai.port", "8000",
# "--parallelism", "8"
# ],
cmds=['python3'],
arguments=['/antispam-batch.py',
"--opensearch.host", conn.host,
"--opensearch.port", str(conn.port),
"--openai.host", "local-ai.kg-airflow.dev-1.eosc.intranet",
"--openai.port", "8000",
"--parallelism", "8"
],
secrets=secrets,
is_delete_operator_pod=True,
in_cluster=True,