From 117d6c1cc41b2fd09dcecf0bb52bda42d7371dcc Mon Sep 17 00:00:00 2001 From: Jose Date: Mon, 6 May 2019 12:35:52 +0200 Subject: [PATCH 1/2] Hopefully fix Travis tests Travis Python 3.6 Add PIPENV_IGNORE_VIRTUALENVS=1 in .travis.yml Upgrade pip in travis Upgrade setuptools in travis Intall requests travis Another approach for urllib3 error Another approach for urllib3 error Another approach for urllib3 error Another approach for urllib3 error Another approach for urllib3 error Reverting changes due to Travis issue --- .gitignore | 3 +- .travis.yml | 14 +-- LICENSE | 2 +- README.md | 13 ++- defaults/main.yml | 2 +- meta/main.yml | 5 +- molecule.yml | 95 ------------------- molecule/default/Dockerfile.j2 | 9 ++ .../default}/files/sample-dag.py | 0 .../default}/group_vars/airflow/main.yml | 0 molecule/default/molecule.yml | 33 +++++++ molecule/default/playbook.yml | 6 ++ .../default/tests/test_airflow.yml | 0 molecule/default/verify.yml | 56 +++++++++++ test-requirements.txt | 3 + tests/goss/verifier.yml | 31 ------ tests/playbook.yml | 11 --- 17 files changed, 125 insertions(+), 158 deletions(-) delete mode 100644 molecule.yml create mode 100644 molecule/default/Dockerfile.j2 rename {tests => molecule/default}/files/sample-dag.py (100%) rename {tests => molecule/default}/group_vars/airflow/main.yml (100%) create mode 100644 molecule/default/molecule.yml create mode 100644 molecule/default/playbook.yml rename tests/goss/specs/airflow.yml.j2 => molecule/default/tests/test_airflow.yml (100%) create mode 100644 molecule/default/verify.yml create mode 100644 test-requirements.txt delete mode 100644 tests/goss/verifier.yml delete mode 100644 tests/playbook.yml diff --git a/.gitignore b/.gitignore index 8d6e156..0c47a6c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ tests/__pycache__ .molecule .cache .vagrant -tests/roles \ No newline at end of file +tests/roles +Pipfile* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 41116cd..0dd618e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,19 @@ --- language: python -python: "2.7.13" +python: "2.7.15" sudo: required -group: deprecated-2017Q4 + +env: + - PIPENV_IGNORE_VIRTUALENVS=1 services: - docker install: - - pip install ansible==2.4.2.0 - - pip install molecule==1.25.0 - - pip install ansible-lint==3.4.20 - - pip install docker + - pip install pipenv==2018.11.26 + - pipenv install -r test-requirements.txt script: - - molecule test --driver docker + - pipenv run molecule test notifications: webhooks: https://galaxy.ansible.com/api/v1/notifications/ diff --git a/LICENSE b/LICENSE index f0b580e..e55af26 100644 --- a/LICENSE +++ b/LICENSE @@ -54,7 +54,7 @@ END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work -Copyright 2018 Idealista S.A.U. +Copyright 2019 Idealista S.A.U. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 0c5f3c5..bb0f140 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,14 @@ This ansible role installs a Apache Airflow server in a Debian/Ubuntu environmen ## Getting Started -These instructions will get you a copy of the role for your ansible playbook. Once launched, it will install a [Apache Airflow](https://airflow.incubator.apache.org/) server in a Debian or Ubuntu system. +These instructions will get you a copy of the role for your ansible playbook. Once launched, it will install a [Apache Airflow](https://airflow.apache.org/) server in a Debian or Ubuntu system. ### Prerequisities -Ansible 2.4.1.0 version installed. +Ansible 2.4.5.0 version installed. Inventory destination should be a Debian or Ubuntu environment. -For testing purposes, [Molecule](https://molecule.readthedocs.io/) with [Vagrant](https://www.vagrantup.com/) as driver (with [landrush](https://github.com/vagrant-landrush/landrush) plugin) and [VirtualBox](https://www.virtualbox.org/) as provider. +For testing purposes, [Molecule](https://molecule.readthedocs.io/) with [Docker](https://www.docker.com/) as driver. ### Installing @@ -75,14 +75,13 @@ airflow_required_python_packages: ## Testing ``` -molecule test --platform=Debian9 +pipenv install -r test-requirements.txt --python 2.7 +pipenv run molecule test ``` -See molecule.yml to check possible testing platforms. - ## Built With -![Ansible](https://img.shields.io/badge/ansible-2.4.1.0-green.svg) +![Ansible](https://img.shields.io/badge/ansible-2.4.5.0-green.svg) ## Versioning diff --git a/defaults/main.yml b/defaults/main.yml index af9e614..4ae2a20 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -10,7 +10,7 @@ airflow_required_libs: - python-pip - acl -airflow_required_python_packages: [] +airflow_required_python_packages: # Version is not mandatory # - {name: pyasn1, version: 0.4.4} diff --git a/meta/main.yml b/meta/main.yml index 0b14382..0a550c0 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -2,12 +2,9 @@ galaxy_info: company: Idealista S.A.U. description: Apache Airflow role - min_ansible_version: 2.4.1.0 + min_ansible_version: 2.4.5.0 license: Apache 2.0 platforms: - name: Debian versions: - stretch - - name: Ubuntu - versions: - - xenial diff --git a/molecule.yml b/molecule.yml deleted file mode 100644 index df31c34..0000000 --- a/molecule.yml +++ /dev/null @@ -1,95 +0,0 @@ ---- -molecule: - # directory in CWD to place all temp files, etc. - molecule_dir: .molecule - - # where temporary state will be stored (lives under molecule_dir) - state_file: state - - # name of temporary vagrantfile created during runs (lives under molecule_dir) - vagrantfile_file: vagrantfile - - # directories to ignore when doing trailing whitespace checks on files during verify command - ignore_paths: - - .git - - .vagrant - - .molecule - - # directory to look for goss tests - goss_dir: tests/goss - goss_playbook: verifier.yml - - # directory containing group_vars to use with ansible - group_vars: ../tests/group_vars - -dependency: - name: galaxy - -# ansible related configuration -ansible: - playbook: tests/playbook.yml - -test: - sequence: - - destroy - - syntax - - create - - converge - - verify - - idempotence - -# configuration options for the internal call to vagrant -vagrant: - raw_config_args: - - "landrush.enabled = true" - - "landrush.tld = 'vm'" - - "landrush.guest_redirect_dns = true" - - # molecule's --platform option will look for these names - platforms: - - name: Debian9 - box: debian/stretch64 - - name: Xenial - box: ubuntu/xenial64 - - providers: - - name: virtualbox - type: virtualbox - options: - memory: 1024 - cpus: 4 - - instances: - - name: airflow.vm - ansible_groups: - - airflow - interfaces: - - network_name: private_network - type: dhcp - auto_config: true - -docker: - containers: - - name: airflow - ansible_groups: - - airflow - - image: geerlingguy/docker-debian9-ansible - image_version: latest - - environment: - TMPDIR: /root/ - - port_bindings: - 8080: 8888 - - # All these parameters are needed for Docker testing with systemd - privileged: True - cap_add: - - SYS_ADMIN - volume_mounts: - - '/sys/fs/cgroup:/sys/fs/cgroup:ro' - command: '/lib/systemd/systemd' - -verifier: - name: goss diff --git a/molecule/default/Dockerfile.j2 b/molecule/default/Dockerfile.j2 new file mode 100644 index 0000000..19124b5 --- /dev/null +++ b/molecule/default/Dockerfile.j2 @@ -0,0 +1,9 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y python sudo bash ca-certificates systemd systemd-sysv && apt-get clean; fi \ No newline at end of file diff --git a/tests/files/sample-dag.py b/molecule/default/files/sample-dag.py similarity index 100% rename from tests/files/sample-dag.py rename to molecule/default/files/sample-dag.py diff --git a/tests/group_vars/airflow/main.yml b/molecule/default/group_vars/airflow/main.yml similarity index 100% rename from tests/group_vars/airflow/main.yml rename to molecule/default/group_vars/airflow/main.yml diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml new file mode 100644 index 0000000..cd82cf2 --- /dev/null +++ b/molecule/default/molecule.yml @@ -0,0 +1,33 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint + enabled: False + +platforms: + - name: airflow-${MOLECULE_DISTRO:-stretch} + image: debian:${MOLECULE_DISTRO:-stretch} + privileged: true + capabilities: + - SYS_ADMIN + volumes: + - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + groups: + - airflow + command: '/lib/systemd/systemd' + +provisioner: + name: ansible + lint: + name: ansible-lint + enabled: False +scenario: + name: default +verifier: + name: goss + lint: + name: yamllint + enabled: False diff --git a/molecule/default/playbook.yml b/molecule/default/playbook.yml new file mode 100644 index 0000000..08ce2b5 --- /dev/null +++ b/molecule/default/playbook.yml @@ -0,0 +1,6 @@ +--- + +- name: Converge + hosts: all + roles: + - role: airflow-role \ No newline at end of file diff --git a/tests/goss/specs/airflow.yml.j2 b/molecule/default/tests/test_airflow.yml similarity index 100% rename from tests/goss/specs/airflow.yml.j2 rename to molecule/default/tests/test_airflow.yml diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml new file mode 100644 index 0000000..17af3ba --- /dev/null +++ b/molecule/default/verify.yml @@ -0,0 +1,56 @@ +--- +# This is an example playbook to execute goss tests. +# Tests need distributed to the appropriate ansible host/groups +# prior to execution by `goss validate`. + +- name: Verify + hosts: all + become: true + vars: + goss_version: v0.3.6 + goss_arch: amd64 + goss_dst: /usr/local/bin/goss + goss_url: "https://github.com/aelsabbahy/goss/releases/download/{{ goss_version }}/goss-linux-{{ goss_arch }}" + goss_test_directory: /tmp + goss_format: documentation + + vars_files: + - ../../defaults/main.yml + - ./group_vars/airflow/main.yml + + tasks: + - name: Download and install Goss + get_url: + url: "{{ goss_url }}" + dest: "{{ goss_dst }}" + mode: 0755 + register: download_goss + until: download_goss is succeeded + retries: 3 + + - name: Copy Goss tests to remote + template: + src: "{{ item }}" + dest: "{{ goss_test_directory }}/{{ item | basename }}" + with_fileglob: + - "{{ lookup('env', 'MOLECULE_VERIFIER_TEST_DIRECTORY') }}/test_*.yml" + + - name: Register test files + shell: "ls {{ goss_test_directory }}/test_*.yml" + register: test_files + + - name: Execute Goss tests + command: "{{ goss_dst }} -g {{ item }} validate --format {{ goss_format }} --retry-timeout 2" + register: test_results + with_items: "{{ test_files.stdout_lines }}" + + - name: Display details about the Goss results + debug: + msg: "{{ item.stdout_lines }}" + with_items: "{{ test_results.results }}" + + - name: Fail when tests fail + fail: + msg: "Goss failed to validate" + when: item.rc != 0 + with_items: "{{ test_results.results }}" diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..4aeddf3 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,3 @@ +ansible==2.4.5.0 +molecule==2.20.1 +docker==3.7.2 \ No newline at end of file diff --git a/tests/goss/verifier.yml b/tests/goss/verifier.yml deleted file mode 100644 index 54bf2bc..0000000 --- a/tests/goss/verifier.yml +++ /dev/null @@ -1,31 +0,0 @@ -- name: Testing prerequisites - hosts: airflow - gather_facts: yes - - vars: - goss_version: "v0.3.6" - goss_url: "https://github.com/aelsabbahy/goss/releases/download/{{ goss_version }}/goss-linux-amd64" - goss_results: goss_results - - vars_files: - - ../../defaults/main.yml - - ../group_vars/airflow/main.yml - - tasks: - - name: Download and install Goss - get_url: - url: "{{ goss_url }}" - dest: "/usr/local/bin/goss" - mode: 0755 - - - name: Copy tests to remote - template: - src: "{{ playbook_dir }}/specs/{{ item }}.j2" - dest: "/tmp/{{ item }}" - with_items: - - airflow.yml - - - name: Goss tests - command: "goss -g /tmp/{{ item }} validate --format tap --retry-timeout 2" - with_items: - - airflow.yml diff --git a/tests/playbook.yml b/tests/playbook.yml deleted file mode 100644 index c6a7037..0000000 --- a/tests/playbook.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- - -- hosts: airflow - roles: - - airflow-role - post_tasks: - # This is to avoid the bug in web UI when no DAG is present - - name: Copy sample-dag.py - copy: - dest: "{{ airflow_dags_folder }}/sample-dag.py" - src: sample-dag.py \ No newline at end of file From 05458a91a4816258d12039f9eac4bbf40c182ce5 Mon Sep 17 00:00:00 2001 From: Jose Date: Tue, 7 May 2019 18:18:18 +0200 Subject: [PATCH 2/2] #56 Fix rebasing collaterals --- defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 4ae2a20..af9e614 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -10,7 +10,7 @@ airflow_required_libs: - python-pip - acl -airflow_required_python_packages: +airflow_required_python_packages: [] # Version is not mandatory # - {name: pyasn1, version: 0.4.4}