giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-03-24 18:56:43 +01:00
parent d97972b85e
commit 8bea0251f1
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
airflow/dags/import_EOSC_graph.py
View File

@ -29,7 +29,7 @@ OPENSEARCH_URL= Variable.get("OPENSEARCH_URL", "https://opensearch-cluster.lot1-
OPENSEARCH_USER = Variable.get("OPENSEARCH_USER", "admin") OPENSEARCH_USER = Variable.get("OPENSEARCH_USER", "admin")
OPENSEARCH_PASSWD = Variable.get("OPENSEARCH_PASSWORD", "admin") OPENSEARCH_PASSWD = Variable.get("OPENSEARCH_PASSWORD", "admin")
ENTITIES = ["datasources", "grants", "organizations", "persons", "products", "topics", "venues"] ENTITIES = ["datasources"] #, "grants", "organizations", "persons", "products", "topics", "venues"]
BULK_PARALLELISM = 10 BULK_PARALLELISM = 10
@ -185,6 +185,7 @@ def import_EOSC_graph():
data = json.loads(line) data = json.loads(line)
data['_index'] = entity data['_index'] = entity
data['_id'] = data['local_identifier'] data['_id'] = data['local_identifier']
data['eoscId'] = ''
yield data yield data
# disable success post logging # disable success post logging