giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-03-27 12:57:02 +01:00
parent 6aa4108b2d
commit f6fbce36e1
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
airflow/dags/import_EOSC_catalog.py
View File

@ -88,12 +88,11 @@ def import_EOSC_catalog():
num_batches = len(pieces)//kwargs["params"]["BATCH_LOADERS_NUM"]
if num_batches > 0:
return list(split_list(pieces, num_batches))
return list(split_list(pieces, len(pieces)))
return [*split_list(pieces, num_batches)]
return [*split_list(pieces, len(pieces))]
@task
def bulk_load(**kwargs):
files = kwargs["files"]
def bulk_load(files: list[(str, str)], **kwargs):
conn = BaseHook.get_connection(kwargs["params"]["OPENSEARCH_CONN_ID"])
client = OpenSearch(
hosts=[{'host': conn.host, 'port': conn.port}],