giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial stage

This commit is contained in:
Giambattista Bloisi 2024-06-27 18:24:24 +02:00
parent a4c75d072b
commit b0f8161e80
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
airflow/dags/EOSC_entity_trasform.py
View File

@ -115,18 +115,23 @@ def isEmpty(current_value: Dict[str, Any], labels: List[str]) -> bool:
#
def filter_product(p: dict) -> bool:
if isEmpty(p, ["titles", "none"]):
print("missing title")
return True
if isEmpty(p, ["firstPublishDate"]):
print("missing date")
return True
if p['product_type'] == "literature":
if isEmpty(p, ["abstracts", "none"]):
print("missing abstracts")
return True
if isEmpty(p, ["contributions", "person", "local_identifier"]):
print("missing creators")
return True
elif p['product_type'] in ["research data", "other"]:
if isEmpty(p, ["contributions", "person", "local_identifier"]):
print("missing creators")
return True
return False
@ -135,3 +140,5 @@ def filter_product(p: dict) -> bool:
filter_entities = {
"products": filter_product
}