giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

simple test DAG

This commit is contained in:
Giambattista Bloisi 2024-03-20 15:57:36 +01:00
parent 620c6fadea
commit dead48e9b2
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
airflow/dags/import_skg_if.py
View File

@ -28,9 +28,9 @@ OPENSEARCH_URL= Variable.get("OPENSEARCH_URL", "https://opensearch-cluster.lot1-
OPENSEARCH_USER = Variable.get("OPENSEARCH_USER", "admin")
OPENSEARCH_PASSWD = Variable.get("OPENSEARCH_PASSWORD", "admin")
ENTITIES = [ "datasources", "grants", "organizations", "persons", "products", "topics", "venues"]
ENTITIES = ["datasources", "grants", "organizations", "persons", "products", "topics", "venues"]
BULK_PARALLELISM = 7
BULK_PARALLELISM = 16
#
@ -171,7 +171,7 @@ def skg_if_pipeline():
for success, item in helpers.parallel_bulk(client, actions=_generate_data(),
raise_on_exception=False,
raise_on_error=False,
chunk_size=500, max_chunk_bytes=10 * 1024 * 1024,
chunk_size=5000, max_chunk_bytes=50 * 1024 * 1024,
timeout=180):
if success:
succeeded = succeeded + 1