Skip to content

Commit

Permalink
Disable daily run of master dag for integration tests (#2130)
Browse files Browse the repository at this point in the history
Since we're not actively developing here, it makes sense to turn off 
the daily schedule of the master DAG (integration tests) to save up 
on the cloud resources that get spunned off from the master DAG.

We do have example DAG run in CI for each new PR, and additionally 
mechanism to tests RCs to guarantee catching of regressions.
  • Loading branch information
pankajkoti authored Mar 21, 2024
1 parent a2fa880 commit 9a5dd6b
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 9 deletions.
14 changes: 6 additions & 8 deletions python-sdk/tests_integration/astro_deploy/master_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import os
import time
from datetime import datetime
from typing import Any, List
from typing import TYPE_CHECKING, Any, List

from airflow import DAG, settings
from airflow.models import Connection, DagRun
Expand All @@ -14,11 +14,13 @@
from airflow.providers.slack.operators.slack_webhook import SlackWebhookOperator
from airflow.utils.session import create_session

if TYPE_CHECKING:
from airflow.models.taskinstance import TaskInstance


SLACK_CHANNEL = os.getenv("SLACK_CHANNEL", "#provider-alert")
SLACK_WEBHOOK_CONN = os.getenv("SLACK_WEBHOOK_CONN", "http_slack")
SLACK_USERNAME = os.getenv("SLACK_USERNAME", "airflow_app")
IS_RUNTIME_RELEASE = os.getenv("IS_RUNTIME_RELEASE", default="False")
IS_RUNTIME_RELEASE = bool(IS_RUNTIME_RELEASE)
AWS_S3_CREDS = {
"aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID", "not_set"),
"aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY", "not_set"),
Expand Down Expand Up @@ -243,13 +245,9 @@ def terminate_instance(task_instance: "TaskInstance") -> None: # noqa: F821
)


if IS_RUNTIME_RELEASE:
schedule_interval = None
else:
schedule_interval = "@daily"
with DAG(
dag_id="example_master_dag",
schedule_interval=schedule_interval,
schedule_interval=None,
start_date=datetime(2023, 1, 1),
catchup=False,
tags=["master_dag"],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def prepare_dag_dependency(task_info, execution_time):

with DAG(
dag_id="example_master_dag_single_worker",
schedule_interval="@daily",
schedule_interval=None,
start_date=datetime(2023, 1, 1),
catchup=False,
tags=["master_dag_single_worker"],
Expand Down

0 comments on commit 9a5dd6b

Please sign in to comment.