lot1-kickoff/airflow/dags/EOSC_entity_trasform.py

74 lines
2.3 KiB
Python

def map_access_right(ar: str) -> str:
match ar:
case 'open':
return 'Open Access'
case 'closed':
return 'Closed'
case 'embargo':
return 'Embargo'
case 'restricted':
return 'Restricted'
case _:
return ''
def trasform_graph_entity(p: dict) -> dict:
p['_id'] = p['local_identifier']
return p
def trasform_catalog_entity(p: dict) -> dict:
p['_id'] = p['id']
return p
def map_fos_topic_to_domain(fos: str):
res = dict()
if fos.startswith('01'):
res['domain'] = 'Natural Sciences'
if fos.startswith('02'):
res['domain'] = 'Engineering & Technology'
if fos.startswith('03'):
res['domain'] = 'Medical & Health Sciences'
if fos.startswith('04'):
res['domain'] = 'Agricultural Sciences'
if fos.startswith('05'):
res['domain'] = 'Social Sciences'
if fos.startswith('06'):
res['domain'] = 'Humanities'
return res
def trasform_product(p: dict) -> dict:
p = trasform_graph_entity(p)
p['accessRights'] = list(
filter(lambda ar: ar != '', map(lambda m: map_access_right(m.get('access_right')), p.get('manifestations'))))
p['keyword'] = list(
map(lambda topic: topic.get('value'),
filter(lambda topic: topic.get('topic').get('scheme') == 'keyword', p.get('topics'))))
p['domain'] = list(filter(lambda fos: fos is not None,
map(lambda topic: map_fos_topic_to_domain(topic.get('value')),
filter(lambda topic: topic.get('topic').get('scheme') == 'FOS', p.get('topics')))))
p['firstPublishDate'] = next(
iter(sorted(filter(lambda date: date.get('type') == 'publishing', p.get('manifestations').get('dates')))),
None)
return p
transform_entities = {
# SKG-IF graph entities
"datasource": trasform_graph_entity,
"grants": trasform_graph_entity,
"organizations": trasform_graph_entity,
"persons": trasform_graph_entity,
"products": trasform_product,
"topics": trasform_graph_entity,
"venues": trasform_graph_entity,
# EOSC catalog entities
"interoperability": trasform_catalog_entity,
"services": trasform_catalog_entity,
"training": trasform_catalog_entity,
}