initial stage

airflow/dags/EOSC_entity_trasform.py

@@ -116,23 +116,18 @@ def isEmpty(current_value: Dict[str, Any], labels: List[str]) -> bool:
 #
 def filter_product(p: dict) -> bool:
     if isEmpty(p, ["titles", "none"]):
-        print("missing title")
         return True
 
     if isEmpty(p, ["firstPublishDate"]):
-        print("missing date")
         return True
 
     if p['product_type'] == "literature":
         if isEmpty(p, ["abstracts", "none"]):
-            print("missing abstracts")
             return True
         if isEmpty(p, ["contributions", "person", "local_identifier"]):
-            print("missing creators")
             return True
     elif p['product_type'] in ["research data", "other"]:
         if isEmpty(p, ["contributions", "person", "local_identifier"]):
-            print("missing creators")
             return True
 
     return False

airflow/dags/import_EOSC_graph.py

@@ -185,7 +185,7 @@ for config_name, config in configs.items():
                                                                raise_on_error=False,
                                                                chunk_size=5000,
                                                                max_chunk_bytes=50 * 1024 * 1024,
-                                                               timeout=180):
+                                                               timeout="5m"):
                         if success:
                             succeeded = succeeded + 1
                         else: