giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-03-26 11:03:05 +01:00
parent 7e41f71d32
commit 26c2e3eaad
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
airflow/dags/common.py
View File

@ -39,7 +39,7 @@ def s3_dowload_unzip_upload(s3conn: str, src_key: str, src_bucket: str, dest_buc
for root, _, files in os.walk(tmp_dir):
for file in files:
local_file_path = os.path.join(root, file)
hook.load_file(local_file_path, strip_prefix(local_file_path, tmp_dir), dest_bucket,
hook.load_file(local_file_path, strip_prefix(local_file_path, tmp_dir + "/"), dest_bucket,
replace=True)

5
airflow/dags/import_EOSC_catalog.py
View File

@ -91,7 +91,10 @@ def import_EOSC_catalog():
for i in range(0, len(list_a), chunk_size):
yield {"files": list_a[i:i + chunk_size]}
return list(split_list(pieces, len(pieces)//BULK_PARALLELISM))
num_batches = len(pieces)//BULK_PARALLELISM
if num_batches > 0:
return list(split_list(pieces, len(pieces)//BULK_PARALLELISM))
return pieces
@task
def bulk_load(files: list[(str, str)], **kwargs):