diff --git a/airflow/dags/OpenDataPortal_harvest.py b/airflow/dags/OpenDataPortal_harvest.py index 073a701..89b3b09 100644 --- a/airflow/dags/OpenDataPortal_harvest.py +++ b/airflow/dags/OpenDataPortal_harvest.py @@ -80,7 +80,7 @@ def harvest(): print(item["index"]["error"]) failed = failed + 1 headers = {'Accept': 'application/json'} - r = requests.get('https://data.europa.eu/api/hub/search/search?filter=dataset&aggregation=false&limit=250&showScore=true&scroll=true', headers=headers).json() + r = requests.get('https://data.europa.eu/api/hub/search/search?filter=dataset&aggregation=false&limit=300&showScore=true&scroll=true', headers=headers).json() scroll_id = r['result']['scrollId'] results = r['result']['results'] store_results(results)