giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-03-24 19:01:00 +01:00
parent 8bea0251f1
commit 99ef9b3980
2 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
airflow/dags/import_EOSC_graph.py
View File

@ -29,7 +29,7 @@ OPENSEARCH_URL= Variable.get("OPENSEARCH_URL", "https://opensearch-cluster.lot1-
OPENSEARCH_USER = Variable.get("OPENSEARCH_USER", "admin")
OPENSEARCH_PASSWD = Variable.get("OPENSEARCH_PASSWORD", "admin")
ENTITIES = ["datasources"] #, "grants", "organizations", "persons", "products", "topics", "venues"]
ENTITIES = ["datasource", "grants", "organizations", "persons", "products", "topics", "venues"]
BULK_PARALLELISM = 10
@ -185,7 +185,6 @@ def import_EOSC_graph():
data = json.loads(line)
data['_index'] = entity
data['_id'] = data['local_identifier']
data['eoscId'] = ''
yield data
# disable success post logging

2
airflow/dags/opensearch_indexes.py
View File

@ -1,7 +1,7 @@
mappings = {}
mappings['datasources'] = {
mappings['datasource'] = {
"properties": {
"data_source_classification": {
"type": "keyword"