initial stage

2024-03-27 12:37:08 +01:00 · 2024-03-27 12:37:08 +01:00 · ab5c8a4b7f
parent 08ed592711
commit ab5c8a4b7f
1 changed files with 1 additions and 1 deletions
--- a/airflow/dags/import_EOSC_catalog.py
+++ b/airflow/dags/import_EOSC_catalog.py
@ -89,7 +89,7 @@ def import_EOSC_catalog():
        num_batches = len(pieces)//kwargs["params"]["BATCH_LOADERS_NUM"]
        if num_batches > 0:
            return list(split_list(pieces, num_batches))
-        return list({"files": pieces})
+        return list(split_list(pieces, len(pieces)))

    @task
    def bulk_load(files: list[(str, str)], **kwargs):