airflow-docker 0.2.0

An opinionated implementation of exclusively using airflow DockerOperators for all Operators

airflow-docker

Description

An opinionated implementation of exclusively using airflow DockerOperators for all Operators.

Default Operator

from airflow_docker.operator import Operator

task = Operator(
    image='some-image:latest',
    ...
)

Default Sensor

from airflow_docker.operator import Sensor

sensor = Sensor(
    image='some-image:latest',
    ...
)

Task Code

from airflow_docker_helper import client

client.sensor(True)

Branch Operator

Dag Task

from airflow_docker.operator import BranchOperator

branching_task = BranchOperator(
    image='some-image:latest',
    ...
)

Task Code

from airflow_docker_helper import client

client.branch_to_tasks(['task1', 'task2'])

Short Circuit Operator

Dag Task

from airflow_docker.operator import ShortCircuitOperator

short_circuit = ShortCircuitOperator(
    image='some-image:latest',
    ...
)

Task Code

from airflow_docker_helper import client

client.short_circuit()  # This task will short circuit if this function gets called

Context Usage

Dag Task

from airflow_docker.operator import Operator

task = Operator(
    image='some-image:latest',
    provide_context=True,
    ...
)

Task Code

from airflow_docker_helper import client

context = client.context()