giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-04-05 19:09:36 +02:00
parent 3aab558117
commit 8461dc62cc
1 changed files with 17 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
airflow/dags/EOSC_entity_trasform.py
View File

@ -25,33 +25,33 @@ def trasform_catalog_entity(p: dict) -> dict:
def map_fos_topic_to_domain(fos: str): def map_fos_topic_to_domain(fos: str):
res = dict()
if fos.startswith('01'): if fos.startswith('01'):
res['domain'] = 'Natural Sciences' return 'Natural Sciences'
elif fos.startswith('02'): elif fos.startswith('02'):
res['domain'] = 'Engineering & Technology' return 'Engineering & Technology'
elif fos.startswith('03'): elif fos.startswith('03'):
res['domain'] = 'Medical & Health Sciences' return 'Medical & Health Sciences'
elif fos.startswith('04'): elif fos.startswith('04'):
res['domain'] = 'Agricultural Sciences' return 'Agricultural Sciences'
elif fos.startswith('05'): elif fos.startswith('05'):
res['domain'] = 'Social Sciences' return 'Social Sciences'
elif fos.startswith('06'): elif fos.startswith('06'):
res['domain'] = 'Humanities' return 'Humanities'
return res return None
def trasform_product(p: dict) -> dict: def trasform_product(p: dict) -> dict:
p = trasform_graph_entity(p) p = trasform_graph_entity(p)
p['accessRights'] = list( p['accessRights'] = list(set(
filter(lambda ar: ar != '', map(lambda m: map_access_right(m.get('access_right')), p.get('manifestations')))) filter(lambda ar: ar != '', map(lambda m: map_access_right(m.get('access_right')), p.get('manifestations')))))
p['keyword'] = list( p['keyword'] = list(set(
map(lambda topic: topic.get('value'), map(lambda topic: topic.get('topic').get('value'),
filter(lambda topic: topic.get('topic').get('scheme') == 'keyword', p.get('topics')))) filter(lambda topic: topic.get('topic').get('scheme') == 'keyword', p.get('topics')))))
p['domain'] = list(filter(lambda fos: fos is not None, p['domain'] = list(
map(lambda topic: map_fos_topic_to_domain(topic.get('topic').get('value')), map(lambda fos: {"domain": fos},
filter(lambda topic: topic.get('topic').get('scheme') == 'FOS', p.get('topics'))))) set(filter(lambda fos: fos is not None,
map(lambda topic: map_fos_topic_to_domain(topic.get('topic').get('value')),
filter(lambda topic: topic.get('topic').get('scheme') == 'FOS', p.get('topics')))))))
p['firstPublishDate'] = next( p['firstPublishDate'] = next(
iter( iter(
sorted( sorted(
@ -59,7 +59,6 @@ def trasform_product(p: dict) -> dict:
filter(lambda date: date.get('type') == 'publishing', filter(lambda date: date.get('type') == 'publishing',
[date for m in (p.get('manifestations') or []) for date in (m.get('dates') or [])])))), [date for m in (p.get('manifestations') or []) for date in (m.get('dates') or [])])))),
None) None)
return p return p