simple test DAG

airflow/dags/skg_if_pipeline.py

@@ -13,6 +13,7 @@ from airflow.decorators import dag
 from airflow.decorators import task
 from airflow.providers.amazon.aws.hooks.s3 import S3Hook
 from airflow.utils.file import TemporaryDirectory
+from airflow.utils.helpers import chain
 
 S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "skg-if")
 
@@ -101,14 +102,16 @@ def skg_if_pipeline():
             f'https://opensearch-cluster.lot1-opensearch-cluster.svc.cluster.local:9200/{entity}_index/_refresh',
             verify=False)
 
-    unzip_to_s3("dump.zip", S3_BUCKET_NAME)
-    bulk_load("datasources")
-    bulk_load("grants")
-    bulk_load("organizations")
-    bulk_load("persons")
-    bulk_load("products")
-    bulk_load("topics")
+    chain(
+        unzip_to_s3.override(task_id=f"unzip_to_s3")("dump.zip", S3_BUCKET_NAME),
+        bulk_load.override(task_id=f"load_datasources")("datasources"),
+        bulk_load("grants"),
+        bulk_load("organizations"),
+        bulk_load("persons"),
+        bulk_load("products"),
+        bulk_load("topics"),
         bulk_load("venues")
+    )
 
 
 skg_if_pipeline()