diff --git a/.ansible-lint b/.ansible-lint index 8feb26b..850f246 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -5,6 +5,7 @@ exclude_paths: - .github/ - .cache/ - molecule/default + - molecule/aws-ec2 offline: false use_default_rules: true parseable: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfb703e..665c748 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,8 @@ jobs: include: - distro: ubuntu2004 ansible-version: '>=2.11.5' + - distro: rockylinux8 + ansible-version: '>=2.11.5' steps: - name: Check out the codebase diff --git a/.yamllint b/.yamllint index b0b1796..8827676 100644 --- a/.yamllint +++ b/.yamllint @@ -1,6 +1,5 @@ --- # Based on ansible-lint config - extends: default rules: @@ -32,7 +31,3 @@ rules: type: unix trailing-spaces: disable truthy: disable - -ignore: | - .tox/ - venv diff --git a/meta/main.yml b/meta/main.yml index 362cb6d..4c5bc1f 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -11,6 +11,9 @@ galaxy_info: - name: Ubuntu versions: - focal + - name: EL + versions: + - "8" galaxy_tags: - development - system diff --git a/molecule/aws-ec2/INSTALL.rst b/molecule/aws-ec2/INSTALL.rst new file mode 100644 index 0000000..f5a4328 --- /dev/null +++ b/molecule/aws-ec2/INSTALL.rst @@ -0,0 +1,22 @@ +********************************************* +Amazon Web Services driver installation guide +********************************************* + +Requirements +============ + +* An AWS credentials rc file + +Install +======= + +Please refer to the `Virtual environment`_ documentation for installation best +practices. If not using a virtual environment, please consider passing the +widely recommended `'--user' flag`_ when invoking ``pip``. + +.. _Virtual environment: https://virtualenv.pypa.io/en/latest/ +.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site + +.. code-block:: bash + + $ pip install 'molecule-ec2' diff --git a/molecule/aws-ec2/converge.yml b/molecule/aws-ec2/converge.yml new file mode 100644 index 0000000..fc52d57 --- /dev/null +++ b/molecule/aws-ec2/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + become: true + pre_tasks: + - name: Include vars + include_vars: "{{ playbook_dir }}/../../tests/vars/main.yml" + tasks: + - name: "Include ansible-python-install" + include_role: + name: "ansible-python-install" diff --git a/molecule/aws-ec2/create.yml b/molecule/aws-ec2/create.yml new file mode 100644 index 0000000..67fe035 --- /dev/null +++ b/molecule/aws-ec2/create.yml @@ -0,0 +1,323 @@ +--- +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + collections: + - community.aws + - community.crypto + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: '{{ default_run_config | combine(run_config_from_file) }}' + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ec2-user + default_ssh_port: 22 + default_user_data: '' + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: '{{ platforms }}' + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + + - name: Generate local key pairs + openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: + name: "{{ item.image_name }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + ec2_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - block: + - name: Wait for instance creation to complete + async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Wait for public IP + ansible.builtin.pause: + seconds: 120 + + - name: Collect instance configs + set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + + - name: Start SSH pollers + wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait_async is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" + when: ec2_instances_async is changed diff --git a/molecule/aws-ec2/destroy.yml b/molecule/aws-ec2/destroy.yml new file mode 100644 index 0000000..5794772 --- /dev/null +++ b/molecule/aws-ec2/destroy.yml @@ -0,0 +1,139 @@ +--- +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + collections: + - community.aws + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: '{{ default_run_config | combine(run_config_from_file) }}' + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: '{{ platforms }}' + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy ephemeral EC2 instances + ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Write Molecule instance configs + copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" + + - name: Destroy ephemeral security groups (if needed) + ec2_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" diff --git a/molecule/aws-ec2/molecule.yml b/molecule/aws-ec2/molecule.yml new file mode 100644 index 0000000..6bae7fb --- /dev/null +++ b/molecule/aws-ec2/molecule.yml @@ -0,0 +1,17 @@ +--- +dependency: + name: galaxy +driver: + name: ec2 +platforms: + - name: instance + image: ami-0f0f1c02e5e4d9d9f + instance_type: t2.micro + vpc_subnet_id: ${AWS_SUBNET} + tags: + Name: ephemeral_molecule_instance + For: molecule +provisioner: + name: ansible +verifier: + name: ansible diff --git a/molecule/aws-ec2/prepare.yml b/molecule/aws-ec2/prepare.yml new file mode 100644 index 0000000..e1b9818 --- /dev/null +++ b/molecule/aws-ec2/prepare.yml @@ -0,0 +1,10 @@ +--- +- name: Prepare + hosts: all + gather_facts: false + tasks: + - name: Make sure python3 is installed + package: + name: python3 + state: present + become: true diff --git a/molecule/aws-ec2/verify.yml b/molecule/aws-ec2/verify.yml new file mode 100644 index 0000000..42b0c49 --- /dev/null +++ b/molecule/aws-ec2/verify.yml @@ -0,0 +1,11 @@ +--- +# This is an example playbook to execute Ansible tests. + +- name: Verify + hosts: all + gather_facts: false + pre_tasks: + - name: Include vars + include_vars: "{{ playbook_dir }}/../../tests/vars/main.yml" + - name: Include tasks + include: "{{ playbook_dir }}/../../tests/tasks/post.yml" diff --git a/requirements.txt b/requirements.txt index 2caac17..bbf9c91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ ansible ansible-lint +boto3 docker molecule molecule-docker +molecule-ec2 yamllint diff --git a/tasks/install-python.yml b/tasks/debian/main.yml similarity index 100% rename from tasks/install-python.yml rename to tasks/debian/main.yml diff --git a/tasks/main.yml b/tasks/main.yml index ddd1476..f11ff9d 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -4,10 +4,29 @@ ansible.builtin.include_vars: "{{ item }}" with_first_found: - "_{{ ansible_distribution_release }}.yml" + - "_{{ (ansible_distribution + '-' + ansible_distribution_major_version) | lower }}.yml" - "_{{ ansible_distribution | lower }}.yml" - _default.yml tags: - configuration -- name: Include installation tasks - ansible.builtin.include_tasks: install-python.yml +- name: Create download directory + ansible.builtin.file: + path: "{{ downloads_path }}" + state: directory + owner: root + group: root + mode: 0755 + +- name: Include tasks + ansible.builtin.include_tasks: '{{ ansible_os_family | lower }}/main.yml' + +- name: Install python | Upgrade tools + ansible.builtin.command: > + /opt/python/{{ item }}/bin/pip install --upgrade \ + pip setuptools wheel + loop: "{{ python_versions }}" + register: command_result + changed_when: "'Successfully installed' in command_result.stdout" + tags: + - python-install-upgrade-tools diff --git a/tasks/redhat/main.yml b/tasks/redhat/main.yml new file mode 100644 index 0000000..1e25961 --- /dev/null +++ b/tasks/redhat/main.yml @@ -0,0 +1,25 @@ +--- + +- name: Install python | Install dependecies + ansible.builtin.yum: + name: "{{ system_dependecies }}" + +- name: Install python | Download rpm package + ansible.builtin.get_url: + url: "{{ python_download_url }}/python-{{ item }}-1-1.x86_64.rpm" + dest: "{{ downloads_path }}/python-{{ item }}-1-1.x86_64.rpm" + mode: "0644" + loop: "{{ python_versions }}" + tags: + - python-install-download-archives + - python-install-setup-versions + +- name: Install python | Install rpm package + ansible.builtin.yum: + name: "{{ downloads_path }}/python-{{ item }}-1-1.x86_64.rpm" + disable_gpg_check: true + state: present + loop: "{{ python_versions }}" + tags: + - python-install-install-archives + - python-install-setup-versions diff --git a/tests/test.yml b/tests/test.yml index 3c9a06c..5d3d5cf 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -1,13 +1,14 @@ # test file --- -- hosts: localhost +- name: Include vars and tasks + hosts: localhost connection: local become: true pre_tasks: - - name: include vars - include_vars: "{{ playbook_dir }}/vars/main.yml" + - name: Include vars + ansible.builtin.include_vars: "{{ playbook_dir }}/vars/main.yml" roles: - ../../ post_tasks: - - name: include tasks - include: "{{ playbook_dir }}/tasks/post.yml" + - name: Include tasks + ansible.builtin.include_tasks: "{{ playbook_dir }}/tasks/post.yml" diff --git a/vars/_redhat-8.yml b/vars/_redhat-8.yml new file mode 100644 index 0000000..48b2783 --- /dev/null +++ b/vars/_redhat-8.yml @@ -0,0 +1,5 @@ +# vars file +--- +system_dependecies: + - libev-devel +python_download_url: "https://cdn.rstudio.com/python/centos-8/pkgs" diff --git a/vars/_rocky-8.yml b/vars/_rocky-8.yml new file mode 100644 index 0000000..48b2783 --- /dev/null +++ b/vars/_rocky-8.yml @@ -0,0 +1,5 @@ +# vars file +--- +system_dependecies: + - libev-devel +python_download_url: "https://cdn.rstudio.com/python/centos-8/pkgs" diff --git a/vars/main.yml b/vars/main.yml index 48b3e84..ec984c3 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -1,2 +1,4 @@ # vars file --- + +downloads_path: /var/lib/ansible/downloads