giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-04-08 14:25:11 +02:00
parent ba37ed66eb
commit 28d2e96842
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
airflow/dags/import_EOSC_graph.py
View File

@ -154,10 +154,9 @@ for config_name, config in configs.items():
continue
print(f'Processing {indexname}: {key}')
s3_obj = hook.get_key(key, bucket_name=kwargs["params"]["EOSC_CATALOG_BUCKET"])
with gzip.GzipFile(s3_obj.get()["Body"]) if key.endswith(".gz") else codecs.getreader('utf-8')(s3_obj.get()["Body"]) as gzipfile:
with io.BufferedReader(gzip.GzipFile(s3_obj.get()["Body"])) if key.endswith(".gz") else codecs.getreader('utf-8')(s3_obj.get()["Body"]) as s3file:
def _generate_data():
buff = io.BufferedReader(gzipfile)
for line in buff:
for line in s3file:
data = json.loads(line)
data['_index'] = indexname
if entity in transform_entities: