From ded5be3f213112320b44cce9a4963b3d683d7259 Mon Sep 17 00:00:00 2001 From: Jose Date: Tue, 18 Jun 2019 13:36:19 +0200 Subject: [PATCH 1/4] #50 Fix Airflow extra packages scenario and compatibility with more Ansible versions --- .travis.yml | 10 ++-- CHANGELOG.md | 2 +- README.md | 6 ++ defaults/main.yml | 2 + molecule/default/verify.yml | 4 +- molecule/extra_packages/Dockerfile.j2 | 9 +++ molecule/extra_packages/files/sample-dag.py | 33 +++++++++++ .../group_vars/airflow/main.yml | 15 +++++ molecule/extra_packages/molecule.yml | 33 +++++++++++ molecule/extra_packages/playbook.yml | 6 ++ .../extra_packages/tests/test_airflow.yml | 24 ++++++++ molecule/extra_packages/verify.yml | 56 +++++++++++++++++++ tasks/install.yml | 15 ++++- test-requirements.txt | 1 - 14 files changed, 206 insertions(+), 10 deletions(-) create mode 100644 molecule/extra_packages/Dockerfile.j2 create mode 100644 molecule/extra_packages/files/sample-dag.py create mode 100644 molecule/extra_packages/group_vars/airflow/main.yml create mode 100644 molecule/extra_packages/molecule.yml create mode 100644 molecule/extra_packages/playbook.yml create mode 100644 molecule/extra_packages/tests/test_airflow.yml create mode 100644 molecule/extra_packages/verify.yml diff --git a/.travis.yml b/.travis.yml index 0dd618e..e778fd0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,16 +4,18 @@ python: "2.7.15" sudo: required env: - - PIPENV_IGNORE_VIRTUALENVS=1 + - ansible_version: 2.4.5 + - ansible_version: 2.7.9 + - ansible_version: 2.8.1 services: - docker install: - - pip install pipenv==2018.11.26 - - pipenv install -r test-requirements.txt + - pip install ansible==${ansible_version} + - pip install -r test-requirements.txt script: - - pipenv run molecule test + - molecule test --all notifications: webhooks: https://galaxy.ansible.com/api/v1/notifications/ diff --git a/CHANGELOG.md b/CHANGELOG.md index fd48c42..866a689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a ch ## [1.7.4](https://github.com/idealista/airflow-role/tree/1.7.4) [Full Changelog](https://github.com/idealista/airflow-role/compare/1.7.3...1.7.4) ### Fixed -- *[#47](https://github.com/idealista/airflow-role/issues/50) Fix deprecation warning from jinja templates @adrimarteau +- *[#50](https://github.com/idealista/airflow-role/issues/50) Fix deprecation warning from jinja templates @adrimarteau @jnogol ## [1.7.3](https://github.com/idealista/airflow-role/tree/1.7.3) [Full Changelog](https://github.com/idealista/airflow-role/compare/1.7.2...1.7.3) diff --git a/README.md b/README.md index bb0f140..41d0106 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,12 @@ airflow_required_python_packages: - {name: pyasn1, version: 0.4.4} ``` +`airflow_extra_packages` (available at: https://airflow.apache.org/installation.html#extra-packages) should be a list following this format: +``` yml +airflow_extra_packages: + - celery + - mysql +``` ## Testing ``` diff --git a/defaults/main.yml b/defaults/main.yml index af9e614..672696d 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,8 @@ --- ## General airflow_version: 1.9.0 +# Available extra packages: https://airflow.apache.org/installation.html#extra-packages +# List should follow Ansible's YAML basics: https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html#yaml-basics airflow_extra_packages: [] airflow_pip_executable: "pip" diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index 250e66b..4e703ad 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -7,7 +7,7 @@ hosts: all become: true vars: - goss_version: v0.3.6 + goss_version: v0.3.7 goss_arch: amd64 goss_dst: /usr/local/bin/goss goss_url: "https://github.com/aelsabbahy/goss/releases/download/{{ goss_version }}/goss-linux-{{ goss_arch }}" @@ -40,7 +40,7 @@ register: test_files - name: Execute Goss tests - command: "{{ goss_dst }} -g {{ item }} validate --format {{ goss_format }} --retry-timeout 2s" + command: "{{ goss_dst }} -g {{ item }} validate --format {{ goss_format }} --retry-timeout 10s" register: test_results with_items: "{{ test_files.stdout_lines }}" diff --git a/molecule/extra_packages/Dockerfile.j2 b/molecule/extra_packages/Dockerfile.j2 new file mode 100644 index 0000000..19124b5 --- /dev/null +++ b/molecule/extra_packages/Dockerfile.j2 @@ -0,0 +1,9 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y python sudo bash ca-certificates systemd systemd-sysv && apt-get clean; fi \ No newline at end of file diff --git a/molecule/extra_packages/files/sample-dag.py b/molecule/extra_packages/files/sample-dag.py new file mode 100644 index 0000000..1aad890 --- /dev/null +++ b/molecule/extra_packages/files/sample-dag.py @@ -0,0 +1,33 @@ +from datetime import datetime +from airflow import DAG +from airflow.operators.dummy_operator import DummyOperator + +args = { + 'owner': 'airflow', + 'start_date': datetime(2017, 9, 10, 0, 0), + 'random_logic': False +} + +dag = DAG( + 'sample_dag', + schedule_interval="@once", + default_args=args +) + +t1 = DummyOperator( + task_id='extract_data', + dag=dag +) + +t2 = DummyOperator( + task_id='load_data', + dag=dag +) + +t3 = DummyOperator( + task_id='random_task', + dag=dag +) + +t1.set_downstream(t2) +t2.set_downstream(t3) \ No newline at end of file diff --git a/molecule/extra_packages/group_vars/airflow/main.yml b/molecule/extra_packages/group_vars/airflow/main.yml new file mode 100644 index 0000000..7df1068 --- /dev/null +++ b/molecule/extra_packages/group_vars/airflow/main.yml @@ -0,0 +1,15 @@ +--- +airflow_fernet_key: xKy13nPFfDflJ0DYGVTwf_DEmbItfURHlEDxrt-bBQw= + +airflow_required_libs: + - acl + - python-pip + - python-mysqldb + +airflow_extra_packages: + - celery + - mysql + +# airflow_extra_packages: celery, mysql + +# airflow_extra_packages: [celery, mysql] \ No newline at end of file diff --git a/molecule/extra_packages/molecule.yml b/molecule/extra_packages/molecule.yml new file mode 100644 index 0000000..0764729 --- /dev/null +++ b/molecule/extra_packages/molecule.yml @@ -0,0 +1,33 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint + enabled: False + +platforms: + - name: airflow_extra_packages-${MOLECULE_DISTRO:-stretch} + image: debian:${MOLECULE_DISTRO:-stretch} + privileged: true + capabilities: + - SYS_ADMIN + volumes: + - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + groups: + - airflow + command: '/lib/systemd/systemd' + +provisioner: + name: ansible + lint: + name: ansible-lint + enabled: False +scenario: + name: extra_packages +verifier: + name: goss + lint: + name: yamllint + enabled: False diff --git a/molecule/extra_packages/playbook.yml b/molecule/extra_packages/playbook.yml new file mode 100644 index 0000000..08ce2b5 --- /dev/null +++ b/molecule/extra_packages/playbook.yml @@ -0,0 +1,6 @@ +--- + +- name: Converge + hosts: all + roles: + - role: airflow-role \ No newline at end of file diff --git a/molecule/extra_packages/tests/test_airflow.yml b/molecule/extra_packages/tests/test_airflow.yml new file mode 100644 index 0000000..666a275 --- /dev/null +++ b/molecule/extra_packages/tests/test_airflow.yml @@ -0,0 +1,24 @@ +service: +{% for airflow_service in airflow_services %} +{% if airflow_services[airflow_service]["enabled"] %} + {{ airflow_service }}: + enabled: true + running: true +{% endif %} +{% endfor %} + +user: + {{ airflow_user }}: + exists: true + groups: + - {{ airflow_group }} + +group: + {{ airflow_group }}: + exists: true + +command: + /usr/local/bin/airflow version: + exit-status: 0 + stdout: + - "{{ airflow_version }}" \ No newline at end of file diff --git a/molecule/extra_packages/verify.yml b/molecule/extra_packages/verify.yml new file mode 100644 index 0000000..4e703ad --- /dev/null +++ b/molecule/extra_packages/verify.yml @@ -0,0 +1,56 @@ +--- +# This is an example playbook to execute goss tests. +# Tests need distributed to the appropriate ansible host/groups +# prior to execution by `goss validate`. + +- name: Verify + hosts: all + become: true + vars: + goss_version: v0.3.7 + goss_arch: amd64 + goss_dst: /usr/local/bin/goss + goss_url: "https://github.com/aelsabbahy/goss/releases/download/{{ goss_version }}/goss-linux-{{ goss_arch }}" + goss_test_directory: /tmp + goss_format: documentation + + vars_files: + - ../../defaults/main.yml + - ./group_vars/airflow/main.yml + + tasks: + - name: Download and install Goss + get_url: + url: "{{ goss_url }}" + dest: "{{ goss_dst }}" + mode: 0755 + register: download_goss + until: download_goss is succeeded + retries: 3 + + - name: Copy Goss tests to remote + template: + src: "{{ item }}" + dest: "{{ goss_test_directory }}/{{ item | basename }}" + with_fileglob: + - "{{ lookup('env', 'MOLECULE_VERIFIER_TEST_DIRECTORY') }}/test_*.yml" + + - name: Register test files + shell: "ls {{ goss_test_directory }}/test_*.yml" + register: test_files + + - name: Execute Goss tests + command: "{{ goss_dst }} -g {{ item }} validate --format {{ goss_format }} --retry-timeout 10s" + register: test_results + with_items: "{{ test_files.stdout_lines }}" + + - name: Display details about the Goss results + debug: + msg: "{{ item.stdout_lines }}" + with_items: "{{ test_results.results }}" + + - name: Fail when tests fail + fail: + msg: "Goss failed to validate" + when: item.rc != 0 + with_items: "{{ test_results.results }}" diff --git a/tasks/install.yml b/tasks/install.yml index d17f59c..84273eb 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -67,12 +67,23 @@ environment: SLUGIFY_USES_TEXT_UNIDECODE: "yes" +- name: Airflow | Installing Airflow Extra Packages (prior to Ansible 2.7) + pip: + executable: "{{ airflow_pip_executable }}" + name: "apache-airflow[{{ item }}]=={{ airflow_version }}" + with_items: "{{ airflow_extra_packages }}" + when: + - airflow_extra_packages + - ansible_version.full is version_compare('2.7', '<') + - name: Airflow | Installing Airflow Extra Packages pip: executable: "{{ airflow_pip_executable }}" - name: apache-airflow[{{ airflow_extra_packages }}] + name: "apache-airflow[{{ airflow_extra_packages | join(', ') }}]" version: "{{ airflow_version }}" - when: airflow_extra_packages + when: + - airflow_extra_packages + - ansible_version.full is version_compare('2.7', '>=') - name: Airflow | Installing DAGs dependencies pip: diff --git a/test-requirements.txt b/test-requirements.txt index 4aeddf3..723e8e0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,2 @@ -ansible==2.4.5.0 molecule==2.20.1 docker==3.7.2 \ No newline at end of file From 7c92ff9b4a038d013eeea030211ea169e7639624 Mon Sep 17 00:00:00 2001 From: Jose Date: Tue, 18 Jun 2019 13:44:23 +0200 Subject: [PATCH 2/4] #50 Delete comment in test group_vars --- molecule/extra_packages/group_vars/airflow/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/molecule/extra_packages/group_vars/airflow/main.yml b/molecule/extra_packages/group_vars/airflow/main.yml index 7df1068..1ba0c6b 100644 --- a/molecule/extra_packages/group_vars/airflow/main.yml +++ b/molecule/extra_packages/group_vars/airflow/main.yml @@ -9,7 +9,3 @@ airflow_required_libs: airflow_extra_packages: - celery - mysql - -# airflow_extra_packages: celery, mysql - -# airflow_extra_packages: [celery, mysql] \ No newline at end of file From 4b7a3cbe136eec021fbfdc47e909d8b99f6af288 Mon Sep 17 00:00:00 2001 From: Jose Date: Tue, 18 Jun 2019 13:56:32 +0200 Subject: [PATCH 3/4] #61 Add KillSignal=SIGINT to workers service file --- templates/airflow-worker.service.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/airflow-worker.service.j2 b/templates/airflow-worker.service.j2 index f3d63e3..68a1468 100644 --- a/templates/airflow-worker.service.j2 +++ b/templates/airflow-worker.service.j2 @@ -22,6 +22,7 @@ User={{ airflow_user }} Group={{ airflow_group }} Type=simple ExecStart={{ airflow_executable }} worker --pid {{ airflow_pidfile_folder }}/worker.pid +KillSignal=SIGINT Restart=on-failure RestartSec=10s RuntimeDirectory=airflow From 4a7a2e26144f7440ec9707a968157eeffd88b000 Mon Sep 17 00:00:00 2001 From: Jose Date: Tue, 18 Jun 2019 13:59:10 +0200 Subject: [PATCH 4/4] #60 Update changelog --- CHANGELOG.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 866a689..399ec09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,12 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a ch ## [Unreleased](https://github.com/idealista/airflow-role/tree/develop) -## [1.7.4](https://github.com/idealista/airflow-role/tree/1.7.4) -[Full Changelog](https://github.com/idealista/airflow-role/compare/1.7.3...1.7.4) +## [1.8.0](https://github.com/idealista/airflow-role/tree/1.8.0) +[Full Changelog](https://github.com/idealista/airflow-role/compare/1.7.3...1.8.0) +### Added +- *[#61](https://github.com/idealista/airflow-role/issues/61) Add KillSignal=SIGINT to workers service file* @jnogol ### Fixed -- *[#50](https://github.com/idealista/airflow-role/issues/50) Fix deprecation warning from jinja templates @adrimarteau @jnogol +- *[#50](https://github.com/idealista/airflow-role/issues/50) Fix deprecation warning from jinja templates* @adrimarteau @jnogol ## [1.7.3](https://github.com/idealista/airflow-role/tree/1.7.3) [Full Changelog](https://github.com/idealista/airflow-role/compare/1.7.2...1.7.3)