giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

simple test DAG

This commit is contained in:
Giambattista Bloisi 2024-03-20 17:33:12 +01:00
parent a7e485a8c6
commit 10fedb06f1
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
airflow/dags/import_skg_if.py
View File

@ -31,7 +31,7 @@ OPENSEARCH_PASSWD = Variable.get("OPENSEARCH_PASSWORD", "admin")
ENTITIES = ["datasources", "grants", "organizations", "persons", "products", "topics", "venues"] ENTITIES = ["datasources", "grants", "organizations", "persons", "products", "topics", "venues"]
BULK_PARALLELISM = 16 BULK_PARALLELISM = 10
# #
@ -114,15 +114,15 @@ def skg_if_pipeline():
} }
}, },
# "mappings": mappings[entity] "mappings": mappings[entity]
"mappings":{ # "mappings":{
"dynamic": False, # "dynamic": False,
"properties": { # "properties": {
"local_identifier": { # "local_identifier": {
"type": "keyword" # "type": "keyword"
} # }
} # }
} # }
}) })
def compute_batches(ds=None, **kwargs): def compute_batches(ds=None, **kwargs):
@ -148,7 +148,7 @@ def skg_if_pipeline():
name="base", name="base",
resources=k8s.V1ResourceRequirements( resources=k8s.V1ResourceRequirements(
requests={ requests={
"cpu": "1", "cpu": "550m",
"memory": "256Mi" "memory": "256Mi"
} }
) )