giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-03-27 12:37:08 +01:00
parent 08ed592711
commit ab5c8a4b7f
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
airflow/dags/import_EOSC_catalog.py
View File

@ -89,7 +89,7 @@ def import_EOSC_catalog():
num_batches = len(pieces)//kwargs["params"]["BATCH_LOADERS_NUM"] num_batches = len(pieces)//kwargs["params"]["BATCH_LOADERS_NUM"]
if num_batches > 0: if num_batches > 0:
return list(split_list(pieces, num_batches)) return list(split_list(pieces, num_batches))
return list({"files": pieces}) return list(split_list(pieces, len(pieces)))
@task @task
def bulk_load(files: list[(str, str)], **kwargs): def bulk_load(files: list[(str, str)], **kwargs):