-
Notifications
You must be signed in to change notification settings - Fork 3
/
airflow.cfg
1363 lines (1069 loc) · 56.8 KB
/
airflow.cfg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
[core]
# The folder where your airflow pipelines live, most likely a
# subfolder in a code repository. This path must be absolute.
# dags_folder = {AIRFLOW_HOME}
# Hostname by providing a path to a callable, which will resolve the hostname.
# The format is "package.function".
#
# For example, default value "airflow.utils.net.getfqdn" means that result from patched
# version of socket.getfqdn() - see https://github.com/python/cpython/issues/49254.
#
# No argument should be required in the function specified.
# If using IP address as hostname is preferred, use value ``airflow.utils.net.get_host_ip_address``
hostname_callable = socket.getfqdn
# A callable to check if a python file has airflow dags defined or not
# with argument as: `(file_path: str, zip_file: zipfile.ZipFile | None = None)`
# return True if it has dags otherwise False
# If this is not provided, Airflow uses its own heuristic rules.
might_contain_dag_callable = airflow.utils.file.might_contain_dag_via_default_heuristic
# Default timezone in case supplied date times are naive
# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam)
default_timezone = utc
# The executor class that airflow should use. Choices include
# ``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``, ``DaskExecutor``,
# ``KubernetesExecutor``, ``CeleryKubernetesExecutor`` or the
# full import path to the class when using a custom executor.
executor = CeleryExecutor
# This defines the maximum number of task instances that can run concurrently per scheduler in
# Airflow, regardless of the worker count. Generally this value, multiplied by the number of
# schedulers in your cluster, is the maximum number of task instances with the running
# state in the metadata database.
parallelism = 32
# The maximum number of task instances allowed to run concurrently in each DAG. To calculate
# the number of tasks that is running concurrently for a DAG, add up the number of running
# tasks for all DAG runs of the DAG. This is configurable at the DAG level with ``max_active_tasks``,
# which is defaulted as ``max_active_tasks_per_dag``.
#
# An example scenario when this would be useful is when you want to stop a new dag with an early
# start date from stealing all the executor slots in a cluster.
max_active_tasks_per_dag = 16
# Are DAGs paused by default at creation
dags_are_paused_at_creation = False
# The maximum number of active DAG runs per DAG. The scheduler will not create more DAG runs
# if it reaches the limit. This is configurable at the DAG level with ``max_active_runs``,
# which is defaulted as ``max_active_runs_per_dag``.
max_active_runs_per_dag = 16
# The name of the method used in order to start Python processes via the multiprocessing module.
# This corresponds directly with the options available in the Python docs:
# https://docs.python.org/3/library/multiprocessing.html#multiprocessing.set_start_method.
# Must be one of the values returned by:
# https://docs.python.org/3/library/multiprocessing.html#multiprocessing.get_all_start_methods.
# Example: mp_start_method = fork
# mp_start_method =
# Whether to load the DAG examples that ship with Airflow. It's good to
# get started, but you probably want to set this to ``False`` in a production
# environment
load_examples = False
# Path to the folder containing Airflow plugins
# plugins_folder = {AIRFLOW_HOME}/plugins
# Should tasks be executed via forking of the parent process ("False",
# the speedier option) or by spawning a new python process ("True" slow,
# but means plugin changes picked up by tasks straight away)
execute_tasks_new_python_interpreter = False
# Secret key to save connection passwords in the db
fernet_key =
# Whether to disable pickling dags
donot_pickle = True
# How long before timing out a python file import
dagbag_import_timeout = 30.0
# Should a traceback be shown in the UI for dagbag import errors,
# instead of just the exception message
dagbag_import_error_tracebacks = True
# If tracebacks are shown, how many entries from the traceback should be shown
dagbag_import_error_traceback_depth = 2
# How long before timing out a DagFileProcessor, which processes a dag file
dag_file_processor_timeout = 50
# The class to use for running task instances in a subprocess.
# Choices include StandardTaskRunner, CgroupTaskRunner or the full import path to the class
# when using a custom task runner.
task_runner = StandardTaskRunner
# If set, tasks without a ``run_as_user`` argument will be run with this user
# Can be used to de-elevate a sudo user running Airflow when executing tasks
default_impersonation =
# What security module to use (for example kerberos)
security =
# Turn unit test mode on (overwrites many configuration options with test
# values at runtime)
unit_test_mode = False
# Whether to enable pickling for xcom (note that this is insecure and allows for
# RCE exploits).
enable_xcom_pickling = False
# What classes can be imported during deserialization. This is a multi line value.
# The individual items will be parsed as regexp. Python built-in classes (like dict)
# are always allowed. Bare "." will be replaced so you can set airflow.* .
allowed_deserialization_classes = airflow\..*
# When a task is killed forcefully, this is the amount of time in seconds that
# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED
killed_task_cleanup_time = 60
# Whether to override params with dag_run.conf. If you pass some key-value pairs
# through ``airflow dags backfill -c`` or
# ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params.
dag_run_conf_overrides_params = True
# If enabled, Airflow will only scan files containing both ``DAG`` and ``airflow`` (case-insensitive).
dag_discovery_safe_mode = True
# The pattern syntax used in the ".airflowignore" files in the DAG directories. Valid values are
# ``regexp`` or ``glob``.
dag_ignore_file_syntax = regexp
# The number of retries each task is going to have by default. Can be overridden at dag or task level.
default_task_retries = 0
# The number of seconds each task is going to wait by default between retries. Can be overridden at
# dag or task level.
default_task_retry_delay = 300
# The maximum delay (in seconds) each task is going to wait by default between retries.
# This is a global setting and cannot be overridden at task or DAG level.
max_task_retry_delay = 86400
# The weighting method used for the effective total priority weight of the task
default_task_weight_rule = downstream
# The default task execution_timeout value for the operators. Expected an integer value to
# be passed into timedelta as seconds. If not specified, then the value is considered as None,
# meaning that the operators are never timed out by default.
default_task_execution_timeout =
# Updating serialized DAG can not be faster than a minimum interval to reduce database write rate.
min_serialized_dag_update_interval = 30
# If True, serialized DAGs are compressed before writing to DB.
# Note: this will disable the DAG dependencies view
compress_serialized_dags = False
# Fetching serialized DAG can not be faster than a minimum interval to reduce database
# read rate. This config controls when your DAGs are updated in the Webserver
min_serialized_dag_fetch_interval = 10
# Maximum number of Rendered Task Instance Fields (Template Fields) per task to store
# in the Database.
# All the template_fields for each of Task Instance are stored in the Database.
# Keeping this number small may cause an error when you try to view ``Rendered`` tab in
# TaskInstance view for older tasks.
max_num_rendered_ti_fields_per_task = 30
# On each dagrun check against defined SLAs
check_slas = True
# Path to custom XCom class that will be used to store and resolve operators results
# Example: xcom_backend = path.to.CustomXCom
xcom_backend = airflow.models.xcom.BaseXCom
# By default Airflow plugins are lazily-loaded (only loaded when required). Set it to ``False``,
# if you want to load plugins whenever 'airflow' is invoked via cli or loaded from module.
lazy_load_plugins = True
# By default Airflow providers are lazily-discovered (discovery and imports happen only when required).
# Set it to False, if you want to discover providers whenever 'airflow' is invoked via cli or
# loaded from module.
lazy_discover_providers = True
# Hide sensitive Variables or Connection extra json keys from UI and task logs when set to True
#
# (Connection passwords are always hidden in logs)
hide_sensitive_var_conn_fields = True
# A comma-separated list of extra sensitive keywords to look for in variables names or connection's
# extra JSON.
sensitive_var_conn_names =
# Task Slot counts for ``default_pool``. This setting would not have any effect in an existing
# deployment where the ``default_pool`` is already created. For existing deployments, users can
# change the number of slots using Webserver, API or the CLI
default_pool_task_slot_count = 128
# The maximum list/dict length an XCom can push to trigger task mapping. If the pushed list/dict has a
# length exceeding this value, the task pushing the XCom will be failed automatically to prevent the
# mapped tasks from clogging the scheduler.
max_map_length = 1024
# The default umask to use for process when run in daemon mode (scheduler, worker, etc.)
#
# This controls the file-creation mode mask which determines the initial value of file permission bits
# for newly created files.
#
# This value is treated as an octal-integer.
daemon_umask = 0o077
# Class to use as dataset manager.
# Example: dataset_manager_class = airflow.datasets.manager.DatasetManager
# dataset_manager_class =
# Kwargs to supply to dataset manager.
# Example: dataset_manager_kwargs = {"some_param": "some_value"}
# dataset_manager_kwargs =
[database]
# The SqlAlchemy connection string to the metadata database.
# SqlAlchemy supports many different database engines.
# More information here:
# http://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html#database-uri
sql_alchemy_conn = postgresql+psycopg2://airflow:airflow@localhost/airflow
# Extra engine specific keyword args passed to SQLAlchemy's create_engine, as a JSON-encoded value
# Example: sql_alchemy_engine_args = {"arg1": True}
# sql_alchemy_engine_args =
# The encoding for the databases
sql_engine_encoding = utf-8
# Collation for ``dag_id``, ``task_id``, ``key``, ``external_executor_id`` columns
# in case they have different encoding.
# By default this collation is the same as the database collation, however for ``mysql`` and ``mariadb``
# the default is ``utf8mb3_bin`` so that the index sizes of our index keys will not exceed
# the maximum size of allowed index when collation is set to ``utf8mb4`` variant
# (see https://github.com/apache/airflow/pull/17603#issuecomment-901121618).
# sql_engine_collation_for_ids =
# If SqlAlchemy should pool database connections.
sql_alchemy_pool_enabled = True
# The SqlAlchemy pool size is the maximum number of database connections
# in the pool. 0 indicates no limit.
sql_alchemy_pool_size = 5
# The maximum overflow size of the pool.
# When the number of checked-out connections reaches the size set in pool_size,
# additional connections will be returned up to this limit.
# When those additional connections are returned to the pool, they are disconnected and discarded.
# It follows then that the total number of simultaneous connections the pool will allow
# is pool_size + max_overflow,
# and the total number of "sleeping" connections the pool will allow is pool_size.
# max_overflow can be set to ``-1`` to indicate no overflow limit;
# no limit will be placed on the total number of concurrent connections. Defaults to ``10``.
sql_alchemy_max_overflow = 10
# The SqlAlchemy pool recycle is the number of seconds a connection
# can be idle in the pool before it is invalidated. This config does
# not apply to sqlite. If the number of DB connections is ever exceeded,
# a lower config value will allow the system to recover faster.
sql_alchemy_pool_recycle = 1800
# Check connection at the start of each connection pool checkout.
# Typically, this is a simple statement like "SELECT 1".
# More information here:
# https://docs.sqlalchemy.org/en/14/core/pooling.html#disconnect-handling-pessimistic
sql_alchemy_pool_pre_ping = True
# The schema to use for the metadata database.
# SqlAlchemy supports databases with the concept of multiple schemas.
sql_alchemy_schema =
# Import path for connect args in SqlAlchemy. Defaults to an empty dict.
# This is useful when you want to configure db engine args that SqlAlchemy won't parse
# in connection string.
# See https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine.params.connect_args
# sql_alchemy_connect_args =
# Whether to load the default connections that ship with Airflow. It's good to
# get started, but you probably want to set this to ``False`` in a production
# environment
load_default_connections = True
# Number of times the code should be retried in case of DB Operational Errors.
# Not all transactions will be retried as it can cause undesired state.
# Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``.
max_db_retries = 3
# Whether to run alembic migrations during Airflow start up. Sometimes this operation can be expensive,
# and the users can assert the correct version through other means (e.g. through a Helm chart).
# Accepts "True" or "False".
check_migrations = True
[logging]
# The folder where airflow should store its log files.
# This path must be absolute.
# There are a few existing configurations that assume this is set to the default.
# If you choose to override this you may need to update the dag_processor_manager_log_location and
# dag_processor_manager_log_location settings as well.
# base_log_folder = {AIRFLOW_HOME}/logs
# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search.
# Set this to True if you want to enable remote logging.
remote_logging = False
# Users must supply an Airflow connection id that provides access to the storage
# location. Depending on your remote logging service, this may only be used for
# reading logs, not writing them.
remote_log_conn_id =
# Whether the local log files for GCS, S3, WASB and OSS remote logging should be deleted after
# they are uploaded to the remote location.
delete_local_logs = False
# Path to Google Credential JSON file. If omitted, authorization based on `the Application Default
# Credentials
# <https://cloud.google.com/docs/authentication/production#finding_credentials_automatically>`__ will
# be used.
google_key_path =
# Storage bucket URL for remote logging
# S3 buckets should start with "s3://"
# Cloudwatch log groups should start with "cloudwatch://"
# GCS buckets should start with "gs://"
# WASB buckets should start with "wasb" just to help Airflow select correct handler
# Stackdriver logs should start with "stackdriver://"
remote_base_log_folder =
# The remote_task_handler_kwargs param is loaded into a dictionary and passed to __init__ of remote
# task handler and it overrides the values provided by Airflow config. For example if you set
# `delete_local_logs=False` and you provide ``{"delete_local_copy": true}``, then the local
# log files will be deleted after they are uploaded to remote location.
# Example: remote_task_handler_kwargs = {"delete_local_copy": true}
remote_task_handler_kwargs =
# Use server-side encryption for logs stored in S3
encrypt_s3_logs = False
# Logging level.
#
# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``.
logging_level = INFO
# Logging level for celery. If not set, it uses the value of logging_level
#
# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``.
celery_logging_level =
# Logging level for Flask-appbuilder UI.
#
# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``.
fab_logging_level = WARNING
# Logging class
# Specify the class that will specify the logging configuration
# This class has to be on the python classpath
# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG
logging_config_class =
# Flag to enable/disable Colored logs in Console
# Colour the logs when the controlling terminal is a TTY.
colored_console_log = True
# Log format for when Colored logs is enabled
colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s
colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter
# Format of Log line
log_format = [%%(asctime)s] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s
simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s
# Where to send dag parser logs. If "file", logs are sent to log files defined by child_process_log_directory.
dag_processor_log_target = file
# Format of Dag Processor Log line
dag_processor_log_format = [%%(asctime)s] [SOURCE:DAG_PROCESSOR] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s
log_formatter_class = airflow.utils.log.timezone_aware.TimezoneAware
# An import path to a function to add adaptations of each secret added with
# `airflow.utils.log.secrets_masker.mask_secret` to be masked in log messages. The given function
# is expected to require a single parameter: the secret to be adapted. It may return a
# single adaptation of the secret or an iterable of adaptations to each be masked as secrets.
# The original secret will be masked as well as any adaptations returned.
# Example: secret_mask_adapter = urllib.parse.quote
secret_mask_adapter =
# Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter
# Example: task_log_prefix_template = {ti.dag_id}-{ti.task_id}-{execution_date}-{try_number}
task_log_prefix_template =
# Formatting for how airflow generates file names/paths for each task run.
log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log
# Formatting for how airflow generates file names for log
log_processor_filename_template = {{ filename }}.log
# Full path of dag_processor_manager logfile.
# dag_processor_manager_log_location = {AIRFLOW_HOME}/logs/dag_processor_manager/dag_processor_manager.log
# Name of handler to read task instance logs.
# Defaults to use ``task`` handler.
task_log_reader = task
# A comma\-separated list of third-party logger names that will be configured to print messages to
# consoles\.
# Example: extra_logger_names = connexion,sqlalchemy
extra_logger_names =
# When you start an airflow worker, airflow starts a tiny web server
# subprocess to serve the workers local log files to the airflow main
# web server, who then builds pages and sends them to users. This defines
# the port on which the logs are served. It needs to be unused, and open
# visible from the main web server to connect into the workers.
worker_log_server_port = 8793
# Port to serve logs from for triggerer. See worker_log_server_port description
# for more info.
trigger_log_server_port = 8794
# We must parse timestamps to interleave logs between trigger and task. To do so,
# we need to parse timestamps in log files. In case your log format is non-standard,
# you may provide import path to callable which takes a string log line and returns
# the timestamp (datetime.datetime compatible).
# Example: interleave_timestamp_parser = path.to.my_func
# interleave_timestamp_parser =
# Permissions in the form or of octal string as understood by chmod. The permissions are important
# when you use impersonation, when logs are written by a different user than airflow. The most secure
# way of configuring it in this case is to add both users to the same group and make it the default
# group of both users. Group-writeable logs are default in airflow, but you might decide that you are
# OK with having the logs other-writeable, in which case you should set it to `0o777`. You might
# decide to add more security if you do not use impersonation and change it to `0o755` to make it
# only owner-writeable. You can also make it just readable only for owner by changing it to `0o700` if
# all the access (read/write) for your logs happens from the same user.
# Example: file_task_handler_new_folder_permissions = 0o775
# file_task_handler_new_folder_permissions = 0o775
# Permissions in the form or of octal string as understood by chmod. The permissions are important
# when you use impersonation, when logs are written by a different user than airflow. The most secure
# way of configuring it in this case is to add both users to the same group and make it the default
# group of both users. Group-writeable logs are default in airflow, but you might decide that you are
# OK with having the logs other-writeable, in which case you should set it to `0o666`. You might
# decide to add more security if you do not use impersonation and change it to `0o644` to make it
# only owner-writeable. You can also make it just readable only for owner by changing it to `0o600` if
# all the access (read/write) for your logs happens from the same user.
# Example: file_task_handler_new_file_permissions = 0o664
# file_task_handler_new_file_permissions = 0o664
[metrics]
# StatsD (https://github.com/etsy/statsd) integration settings.
# If you want to avoid emitting all the available metrics, you can configure an
# allow list of prefixes (comma separated) to send only the metrics that start
# with the elements of the list (e.g: "scheduler,executor,dagrun")
metrics_allow_list =
# If you want to avoid emitting all the available metrics, you can configure a
# block list of prefixes (comma separated) to filter out metrics that start with
# the elements of the list (e.g: "scheduler,executor,dagrun").
# If metrics_allow_list and metrics_block_list are both configured, metrics_block_list is ignored.
metrics_block_list =
# Enables sending metrics to StatsD.
statsd_on = False
statsd_host = localhost
statsd_port = 8125
statsd_prefix = airflow
# A function that validate the StatsD stat name, apply changes to the stat name if necessary and return
# the transformed stat name.
#
# The function should have the following signature:
# def func_name(stat_name: str) -> str:
stat_name_handler =
# To enable datadog integration to send airflow metrics.
statsd_datadog_enabled = False
# List of datadog tags attached to all metrics(e.g: key1:value1,key2:value2)
statsd_datadog_tags =
# Set to False to disable metadata tags for some of the emitted metrics
statsd_datadog_metrics_tags = True
# If you want to utilise your own custom StatsD client set the relevant
# module path below.
# Note: The module path must exist on your PYTHONPATH for Airflow to pick it up
# statsd_custom_client_path =
# If you want to avoid sending all the available metrics tags to StatsD,
# you can configure a block list of prefixes (comma separated) to filter out metric tags
# that start with the elements of the list (e.g: "job_id,run_id")
# Example: statsd_disabled_tags = job_id,run_id,dag_id,task_id
# statsd_disabled_tags = job_id,run_id
# To enable sending Airflow metrics with StatsD-Influxdb tagging convention.
statsd_influxdb_enabled = True
# Enables sending metrics to OpenTelemetry.
# otel_on = False
# otel_host = localhost
# otel_port = 8889
# otel_prefix = airflow
# otel_interval_milliseconds = 60000
[secrets]
# Full class name of secrets backend to enable (will precede env vars and metastore in search path)
# Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend
backend =
# The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class.
# See documentation for the secrets backend you are using. JSON is expected.
# Example for AWS Systems Manager ParameterStore:
# ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}``
backend_kwargs =
[cli]
# In what way should the cli access the API. The LocalClient will use the
# database directly, while the json_client will use the api running on the
# webserver
api_client = airflow.api.client.local_client
# If you set web_server_url_prefix, do NOT forget to append it here, ex:
# ``endpoint_url = http://localhost:8080/myroot``
# So api will look like: ``http://localhost:8080/myroot/api/experimental/...``
endpoint_url = http://localhost:8080
[debug]
# Used only with ``DebugExecutor``. If set to ``True`` DAG will fail with first
# failed task. Helpful for debugging purposes.
fail_fast = False
[api]
# Enables the deprecated experimental API. Please note that these APIs do not have access control.
# The authenticated user has full access.
#
# .. warning::
#
# This `Experimental REST API <https://airflow.readthedocs.io/en/latest/rest-api-ref.html>`__ is
# deprecated since version 2.0. Please consider using
# `the Stable REST API <https://airflow.readthedocs.io/en/latest/stable-rest-api-ref.html>`__.
# For more information on migration, see
# `RELEASE_NOTES.rst <https://github.com/apache/airflow/blob/main/RELEASE_NOTES.rst>`_
enable_experimental_api = False
# Comma separated list of auth backends to authenticate users of the API. See
# https://airflow.apache.org/docs/apache-airflow/stable/security/api.html for possible values.
# ("airflow.api.auth.backend.default" allows all requests for historic reasons)
auth_backends = airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
# Used to set the maximum page limit for API requests. If limit passed as param
# is greater than maximum page limit, it will be ignored and maximum page limit value
# will be set as the limit
maximum_page_limit = 100
# Used to set the default page limit when limit param is zero or not provided in API
# requests. Otherwise if positive integer is passed in the API requests as limit, the
# smallest number of user given limit or maximum page limit is taken as limit.
fallback_page_limit = 100
# The intended audience for JWT token credentials used for authorization. This value must match on the client and server sides. If empty, audience will not be tested.
# Example: google_oauth2_audience = project-id-random-value.apps.googleusercontent.com
google_oauth2_audience =
# Path to Google Cloud Service Account key file (JSON). If omitted, authorization based on
# `the Application Default Credentials
# <https://cloud.google.com/docs/authentication/production#finding_credentials_automatically>`__ will
# be used.
# Example: google_key_path = /files/service-account-json
google_key_path =
# Used in response to a preflight request to indicate which HTTP
# headers can be used when making the actual request. This header is
# the server side response to the browser's
# Access-Control-Request-Headers header.
access_control_allow_headers = origin, content-type, accept
# Specifies the method or methods allowed when accessing the resource.
access_control_allow_methods = POST, GET, OPTIONS, DELETE
# Indicates whether the response can be shared with requesting code from the given origins.
# Separate URLs with space.
access_control_allow_origins =
[lineage]
# what lineage backend to use
backend =
[atlas]
sasl_enabled = False
host =
port = 21000
username =
password =
[operators]
# The default owner assigned to each new operator, unless
# provided explicitly or passed via ``default_args``
default_owner = airflow
default_cpus = 1
default_ram = 512
default_disk = 512
default_gpus = 0
# Default queue that tasks get assigned to and that worker listen on.
default_queue = default
# Is allowed to pass additional/unused arguments (args, kwargs) to the BaseOperator operator.
# If set to False, an exception will be thrown, otherwise only the console message will be displayed.
allow_illegal_arguments = False
[hive]
# Default mapreduce queue for HiveOperator tasks
default_hive_mapred_queue =
# Template for mapred_job_name in HiveOperator, supports the following named parameters
# hostname, dag_id, task_id, execution_date
# mapred_job_name_template =
[webserver]
# The base url of your website as airflow cannot guess what domain or
# cname you are using. This is used in automated emails that
# airflow sends to point links to the right web server
base_url = http://localhost:8080
# Default timezone to display all dates in the UI, can be UTC, system, or
# any IANA timezone string (e.g. Europe/Amsterdam). If left empty the
# default value of core/default_timezone will be used
# Example: default_ui_timezone = America/New_York
default_ui_timezone = UTC
# The ip specified when starting the web server
web_server_host = 0.0.0.0
# The port on which to run the web server
web_server_port = 8080
# Paths to the SSL certificate and key for the web server. When both are
# provided SSL will be enabled. This does not change the web server port.
web_server_ssl_cert =
# Paths to the SSL certificate and key for the web server. When both are
# provided SSL will be enabled. This does not change the web server port.
web_server_ssl_key =
# The type of backend used to store web session data, can be 'database' or 'securecookie'
# Example: session_backend = securecookie
session_backend = securecookie
# Number of seconds the webserver waits before killing gunicorn master that doesn't respond
web_server_master_timeout = 120
# Number of seconds the gunicorn webserver waits before timing out on a worker
web_server_worker_timeout = 120
# Number of workers to refresh at a time. When set to 0, worker refresh is
# disabled. When nonzero, airflow periodically refreshes webserver workers by
# bringing up new ones and killing old ones.
worker_refresh_batch_size = 1
# Number of seconds to wait before refreshing a batch of workers.
worker_refresh_interval = 6000
# If set to True, Airflow will track files in plugins_folder directory. When it detects changes,
# then reload the gunicorn.
reload_on_plugin_change = False
# Secret key used to run your flask app. It should be as random as possible. However, when running
# more than 1 instances of webserver, make sure all of them use the same ``secret_key`` otherwise
# one of them will error with "CSRF session token is missing".
# The webserver key is also used to authorize requests to Celery workers when logs are retrieved.
# The token generated using the secret key has a short expiry time though - make sure that time on
# ALL the machines that you run airflow components on is synchronized (for example using ntpd)
# otherwise you might get "forbidden" errors when the logs are accessed.
secret_key = j4XID8UpYGxP3WExgVQn8A==
# Number of workers to run the Gunicorn web server
workers = 4
# The worker class gunicorn should use. Choices include
# sync (default), eventlet, gevent. Note when using gevent you might also want to set the
# "_AIRFLOW_PATCH_GEVENT" environment variable to "1" to make sure gevent patching is done as
# early as possible.
worker_class = sync
# Log files for the gunicorn webserver. '-' means log to stderr.
access_logfile = -
# Log files for the gunicorn webserver. '-' means log to stderr.
error_logfile = -
# Access log format for gunicorn webserver.
# default format is %%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s"
# documentation - https://docs.gunicorn.org/en/stable/settings.html#access-log-format
access_logformat =
# Expose the configuration file in the web server. Set to "non-sensitive-only" to show all values
# except those that have security implications. "True" shows all values. "False" hides the
# configuration completely.
expose_config = True
# Expose hostname in the web server
expose_hostname = True
# Expose stacktrace in the web server
expose_stacktrace = True
# Default DAG view. Valid values are: ``grid``, ``graph``, ``duration``, ``gantt``, ``landing_times``
dag_default_view = tree
# Default DAG orientation. Valid values are:
# ``LR`` (Left->Right), ``TB`` (Top->Bottom), ``RL`` (Right->Left), ``BT`` (Bottom->Top)
dag_orientation = LR
# The amount of time (in secs) webserver will wait for initial handshake
# while fetching logs from other worker machine
log_fetch_timeout_sec = 5
# Time interval (in secs) to wait before next log fetching.
log_fetch_delay_sec = 2
# Distance away from page bottom to enable auto tailing.
log_auto_tailing_offset = 30
# Animation speed for auto tailing log display.
log_animation_speed = 1000
# By default, the webserver shows paused DAGs. Flip this to hide paused
# DAGs by default
hide_paused_dags_by_default = False
# Consistent page size across all listing views in the UI
page_size = 100
# Define the color of navigation bar
navbar_color = #fff
# Default dagrun to show in UI
default_dag_run_display_number = 25
# Enable werkzeug ``ProxyFix`` middleware for reverse proxy
enable_proxy_fix = False
# Number of values to trust for ``X-Forwarded-For``.
# More info: https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/
proxy_fix_x_for = 1
# Number of values to trust for ``X-Forwarded-Proto``
proxy_fix_x_proto = 1
# Number of values to trust for ``X-Forwarded-Host``
proxy_fix_x_host = 1
# Number of values to trust for ``X-Forwarded-Port``
proxy_fix_x_port = 1
# Number of values to trust for ``X-Forwarded-Prefix``
proxy_fix_x_prefix = 1
# Set secure flag on session cookie
cookie_secure = False
# Set samesite policy on session cookie
cookie_samesite = Lax
# Default setting for wrap toggle on DAG code and TI log views.
default_wrap = False
# Allow the UI to be rendered in a frame
x_frame_enabled = True
# Send anonymous user activity to your analytics tool
# choose from google_analytics, segment, or metarouter
# analytics_tool =
# Unique ID of your account in the analytics tool
# analytics_id =
# 'Recent Tasks' stats will show for old DagRuns if set
show_recent_stats_for_completed_runs = True
# Update FAB permissions and sync security manager roles
# on webserver startup
update_fab_perms = True
# The UI cookie lifetime in minutes. User will be logged out from UI after
# ``session_lifetime_minutes`` of non-activity
session_lifetime_minutes = 43200
# Sets a custom page title for the DAGs overview page and site title for all pages
# instance_name =
# Whether the custom page title for the DAGs overview page contains any Markup language
instance_name_has_markup = False
# How frequently, in seconds, the DAG data will auto-refresh in graph or grid view
# when auto-refresh is turned on
auto_refresh_interval = 3
# Boolean for displaying warning for publicly viewable deployment
warn_deployment_exposure = True
# Comma separated string of view events to exclude from dag audit view.
# All other events will be added minus the ones passed here.
# The audit logs in the db will not be affected by this parameter.
audit_view_excluded_events = gantt,landing_times,tries,duration,calendar,graph,grid,tree,tree_data
# Comma separated string of view events to include in dag audit view.
# If passed, only these events will populate the dag audit view.
# The audit logs in the db will not be affected by this parameter.
# Example: audit_view_included_events = dagrun_cleared,failed
# audit_view_included_events =
# Boolean for running SwaggerUI in the webserver.
enable_swagger_ui = True
# Boolean for enabling rate limiting on authentication endpoints.
auth_rate_limited = True
# Rate limit for authentication endpoints.
auth_rate_limit = 5 per 40 second
# The caching algorithm used by the webserver. Must be a valid hashlib function name.
# Example: caching_hash_method = sha256
caching_hash_method = md5
[email]
# Configuration email backend and whether to
# send email alerts on retry or failure
# Email backend to use
email_backend = airflow.utils.email.send_email_smtp
# Email connection to use
email_conn_id = smtp_default
# Whether email alerts should be sent when a task is retried
default_email_on_retry = True
# Whether email alerts should be sent when a task failed
default_email_on_failure = True
# File that will be used as the template for Email subject (which will be rendered using Jinja2).
# If not set, Airflow uses a base template.
# Example: subject_template = /path/to/my_subject_template_file
# subject_template =
# File that will be used as the template for Email content (which will be rendered using Jinja2).
# If not set, Airflow uses a base template.
# Example: html_content_template = /path/to/my_html_content_template_file
# html_content_template =
# Email address that will be used as sender address.
# It can either be raw email or the complete address in a format ``Sender Name <[email protected]>``
# Example: from_email = Airflow <[email protected]>
# from_email =
[smtp]
# If you want airflow to send emails on retries, failure, and you want to use
# the airflow.utils.email.send_email_smtp function, you have to configure an
# smtp server here
smtp_host = localhost
smtp_starttls = True
smtp_ssl = False
# Example: smtp_user = airflow
# smtp_user =
# Example: smtp_password = airflow
# smtp_password =
smtp_port = 25
smtp_mail_from = [email protected]
smtp_timeout = 30
smtp_retry_limit = 5
[sentry]
# Sentry (https://docs.sentry.io) integration. Here you can supply
# additional configuration options based on the Python platform. See:
# https://docs.sentry.io/error-reporting/configuration/?platform=python.
# Unsupported options: ``integrations``, ``in_app_include``, ``in_app_exclude``,
# ``ignore_errors``, ``before_breadcrumb``, ``transport``.
# Enable error reporting to Sentry
sentry_on = false
sentry_dsn =
# Dotted path to a before_send function that the sentry SDK should be configured to use.
# before_send =
[local_kubernetes_executor]
# This section only applies if you are using the ``LocalKubernetesExecutor`` in
# ``[core]`` section above
# Define when to send a task to ``KubernetesExecutor`` when using ``LocalKubernetesExecutor``.
# When the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``),
# the task is executed via ``KubernetesExecutor``,
# otherwise via ``LocalExecutor``
kubernetes_queue = kubernetes
[celery_kubernetes_executor]
# This section only applies if you are using the ``CeleryKubernetesExecutor`` in
# ``[core]`` section above
# Define when to send a task to ``KubernetesExecutor`` when using ``CeleryKubernetesExecutor``.
# When the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``),
# the task is executed via ``KubernetesExecutor``,
# otherwise via ``CeleryExecutor``
kubernetes_queue = kubernetes
[celery]
# This section only applies if you are using the CeleryExecutor in
# ``[core]`` section above
# The app name that will be used by celery
celery_app_name = airflow.executors.celery_executor
# The concurrency that will be used when starting workers with the
# ``airflow celery worker`` command. This defines the number of task instances that
# a worker will take, so size up your workers based on the resources on
# your worker box and the nature of your tasks
worker_concurrency = 16
# The maximum and minimum concurrency that will be used when starting workers with the
# ``airflow celery worker`` command (always keep minimum processes, but grow
# to maximum if necessary). Note the value should be max_concurrency,min_concurrency
# Pick these numbers based on resources on worker box and the nature of the task.
# If autoscale option is available, worker_concurrency will be ignored.
# http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale
# Example: worker_autoscale = 16,12
# worker_autoscale =
# Used to increase the number of tasks that a worker prefetches which can improve performance.
# The number of processes multiplied by worker_prefetch_multiplier is the number of tasks
# that are prefetched by a worker. A value greater than 1 can result in tasks being unnecessarily
# blocked if there are multiple workers and one worker prefetches tasks that sit behind long
# running tasks while another worker has unutilized processes that are unable to process the already
# claimed blocked tasks.
# https://docs.celeryproject.org/en/stable/userguide/optimizing.html#prefetch-limits
worker_prefetch_multiplier = 1
# Specify if remote control of the workers is enabled.
# In some cases when the broker does not support remote control, Celery creates lots of
# ``.*reply-celery-pidbox`` queues. You can prevent this by setting this to false.
# However, with this disabled Flower won't work.
# https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/index.html#broker-overview
worker_enable_remote_control = true
# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally
# a sqlalchemy database. Refer to the Celery documentation for more information.
broker_url = redis://localhost:6379/0
# The Celery result_backend. When a job finishes, it needs to update the
# metadata of the job. Therefore it will post a message on a message bus,
# or insert it into a database (depending of the backend)
# This status is used by the scheduler to update the state of the task
# The use of a database is highly recommended
# When not specified, sql_alchemy_conn with a db+ scheme prefix will be used
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings
# Example: result_backend = db+postgresql://postgres:airflow@postgres/airflow
result_backend = db+postgresql://airflow:airflow@localhost/airflow
# Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start
# it ``airflow celery flower``. This defines the IP that Celery Flower runs on
flower_host = 0.0.0.0
# The root URL for Flower
# Example: flower_url_prefix = /flower
flower_url_prefix =
# This defines the port that Celery Flower runs on
flower_port = 5555
# Securing Flower with Basic Authentication
# Accepts user:password pairs separated by a comma
# Example: flower_basic_auth = user1:password1,user2:password2
flower_basic_auth =
# How many processes CeleryExecutor uses to sync task state.
# 0 means to use max(1, number of cores - 1) processes.
sync_parallelism = 0
# Import path for celery configuration options
celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
ssl_active = False
ssl_key =
ssl_cert =
ssl_cacert =
# Celery Pool implementation.
# Choices include: ``prefork`` (default), ``eventlet``, ``gevent`` or ``solo``.
# See:
# https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency
# https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html
pool = prefork