giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-04-08 14:22:56 +02:00
parent 51b695c1b7
commit ba37ed66eb
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
airflow/dags/import_EOSC_graph.py
View File

@ -1,5 +1,6 @@
from __future__ import annotations
import codecs
import gzip
import io
import json
@ -153,7 +154,7 @@ for config_name, config in configs.items():
continue
print(f'Processing {indexname}: {key}')
s3_obj = hook.get_key(key, bucket_name=kwargs["params"]["EOSC_CATALOG_BUCKET"])
with gzip.GzipFile(s3_obj.get()["Body"]) if key.endswith(".gz") else s3_obj.get()["Body"] as gzipfile:
with gzip.GzipFile(s3_obj.get()["Body"]) if key.endswith(".gz") else codecs.getreader('utf-8')(s3_obj.get()["Body"]) as gzipfile:
def _generate_data():
buff = io.BufferedReader(gzipfile)
for line in buff: