diff --git a/CHANGELOG.md b/CHANGELOG.md index 25fd334e..1b8a0e8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## UNRELEASED +### ENHANCEMENTS + +* Add a TOSCA component to load a docker image from a tar archive ([GH-140](https://github.com/ystia/forge/issues/140)) + ### BUG FIXES * Docker installation requires apt-transport-https to work properly on Debian based systems ([GH-147](https://github.com/ystia/forge/issues/147)) diff --git a/org/ystia/docker/ansible/playbooks/create.yaml b/org/ystia/docker/ansible/playbooks/create.yaml index 104f73f1..3131b92a 100644 --- a/org/ystia/docker/ansible/playbooks/create.yaml +++ b/org/ystia/docker/ansible/playbooks/create.yaml @@ -12,13 +12,18 @@ - set_fact: # The baseurl of the RedHat/CentOS docker repository. - redhat_docker_repository: "{{ REPOSITORY_URL if (REPOSITORY_URL != '') else ('https://download.docker.com/linux/centos/7/$basearch/stable') }}" + redhat_docker_repository: "{{ REPOSITORY_URL }}" redhat_docker_gpgkey: "{{ DOCKER_GPGKEY if (DOCKER_GPGKEY != '') else ('https://download.docker.com/linux/centos/gpg') }}" # The baseurl of the Debian/Ubuntu docker repository. - debian_docker_repository: "{{ REPOSITORY_URL if (REPOSITORY_URL != '') else ('deb https://download.docker.com/linux/ubuntu xenial stable') }}" + debian_docker_repository: "{{ REPOSITORY_URL }}" debian_docker_gpgkey: "{{ DOCKER_GPGKEY if (DOCKER_GPGKEY != '') else ('https://download.docker.com/linux/ubuntu/gpg') }}" docker_package_name: "docker-ce{{ '-' if (DOCKER_VERSION != '') else ('') }}{{ DOCKER_VERSION if (DOCKER_VERSION != '') else ('') }}" + - set_fact: + redhat_docker_repository: "https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable" + debian_docker_repository: "deb https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" + when: REPOSITORY_URL == "" + - name: Add Docker Yum repository yum_repository: name: docker diff --git a/org/ystia/docker/ansible/types.yml b/org/ystia/docker/ansible/types.yml index a7809af0..43a407df 100644 --- a/org/ystia/docker/ansible/types.yml +++ b/org/ystia/docker/ansible/types.yml @@ -11,6 +11,7 @@ metadata: template_version: 3.0.0-SNAPSHOT template_author: Ystia +description: Docker types imports: - tosca-normative-types:1.0.0-ALIEN20 - yorc-types:1.1.0 diff --git a/org/ystia/docker/containers/generic/playbooks/create.yaml b/org/ystia/docker/containers/generic/playbooks/create.yaml index 9b5ebe5c..325c9a84 100644 --- a/org/ystia/docker/containers/generic/playbooks/create.yaml +++ b/org/ystia/docker/containers/generic/playbooks/create.yaml @@ -8,6 +8,9 @@ - name: Install python requirements hosts: all become: true + environment: + # Adding /usr/local/bin needed by virtualenv on CentOS 8 + PATH: "{{ ansible_env.PATH }}:/usr/local/bin" tasks: - name: Get python version python_requirements_info: @@ -16,20 +19,52 @@ - name: Get python major version set_fact: python_major_version: "{{pri.python_version | replace('\n', '') | regex_replace('^(\\d+).*', '\\1') }}" - - name: Install pip version compatible with python 2 - easy_install: - name: pip<21.0 + - name: Set python version for pip + set_fact: + python_pip_pkg: "python3-pip" + pip_cmd: "pip3" + when: python_major_version != "2" + - name: Set python 2 version for pip + set_fact: + python_pip_pkg: "python-pip" + pip_cmd: "pip" when: python_major_version == "2" - - name: Install pip - easy_install: - name: pip - state: latest + - name: RedHat - install prerequisites + yum: + name: + - "{{ python_pip_pkg }}" + state: present + update_cache: yes + when: ansible_os_family == 'RedHat' + - name: Debian - install prerequisites + apt: + name: + - "{{ python_pip_pkg }}" + state: present + update_cache: yes + register: apt_res + retries: 5 + delay: 15 + until: apt_res is success + when: ansible_os_family == 'Debian' + - name: Install latest Pip version + pip: + name: "pip" + state: latest + executable: "{{pip_cmd}}" when: python_major_version != "2" + - name: Install latest Pip version compatible with python 2 + pip: + name: "pip<21.0" + state: latest + executable: "{{pip_cmd}}" + when: python_major_version == "2" - name: Install 'virtualenv' package pip: name: virtualenv state: latest + executable: "{{pip_cmd}}" - name: Check docker python dependencies shell: > @@ -38,7 +73,9 @@ - name: Install 'docker' python package pip: - name: docker + name: + - docker + - six virtualenv: /usr/local/docker-py-env - name: Create Docker Container diff --git a/org/ystia/docker/containers/generic/playbooks/delete.yaml b/org/ystia/docker/containers/generic/playbooks/delete.yaml index 01d192b4..24dc40a5 100644 --- a/org/ystia/docker/containers/generic/playbooks/delete.yaml +++ b/org/ystia/docker/containers/generic/playbooks/delete.yaml @@ -11,8 +11,14 @@ vars: ansible_python_interpreter: "/usr/local/docker-py-env/bin/python" tasks: - - name: delete docker container + - name: Delete docker container docker_container: name: "{{CONTAINER_ID}}" state: absent + when: STARTED_CONTAINER_ID is not defined or STARTED_CONTAINER_ID == "" + - name: Delete started docker container + docker_container: + name: "{{STARTED_CONTAINER_ID}}" + state: absent + when: STARTED_CONTAINER_ID is defined and STARTED_CONTAINER_ID != "" diff --git a/org/ystia/docker/containers/generic/playbooks/docker_container_tasks.yaml b/org/ystia/docker/containers/generic/playbooks/docker_container_tasks.yaml index 1aaf7162..a6c25365 100644 --- a/org/ystia/docker/containers/generic/playbooks/docker_container_tasks.yaml +++ b/org/ystia/docker/containers/generic/playbooks/docker_container_tasks.yaml @@ -30,14 +30,17 @@ name: "{{NODE}}-{{INSTANCE}}" auto_remove: "{{AUTO_REMOVE}}" cleanup: "{{CLEANUP}}" + command: "{{COMMAND}}" # cpus: "{{CPU_SHARE}}" detach: "{{DETACH}}" env: "{{DOCKER_ENV}}" exposed_ports: "{{DOCKER_EXP_PORTS}}" + hostname: "{{HOSTNAME}}" image: "{{IMAGE}}" keep_volumes: "{{KEEP_VOLUMES}}" memory: "{{MEM_SHARE_LIMIT}}" memory_reservation: "{{MEM_SHARE}}" + network_mode: "{{NETWORK_MODE}}" published_ports: "{{DOCKER_PUB_PORT}}" restart_policy: "{{RESTART_POLICY}}" shm_size: "{{SHM_SIZE}}" diff --git a/org/ystia/docker/containers/generic/playbooks/start.yaml b/org/ystia/docker/containers/generic/playbooks/start.yaml index 12ec1332..eaa9dfd0 100644 --- a/org/ystia/docker/containers/generic/playbooks/start.yaml +++ b/org/ystia/docker/containers/generic/playbooks/start.yaml @@ -7,6 +7,7 @@ - name: Start Docker Container hosts: all become: true + # When the container fails, go on to retrieve outputs vars: ansible_python_interpreter: "/usr/local/docker-py-env/bin/python" DOCKER_ENV: {} @@ -15,8 +16,24 @@ DOCKER_VOLUMES: [] DOCKER_STATE: "started" tasks: - - import_tasks: docker_container_tasks.yaml - - name: Get container output + - name: Start container + import_tasks: docker_container_tasks.yaml + no_log: true + ignore_errors: yes + - name: Store container id (needed if it was directly started or config changed between create and start) + set_fact: + STARTED_CONTAINER_ID: "{{docker_res.ansible_facts.docker_container['Id']}}" + when: docker_res.ansible_facts is defined + - name: Get container output (truncated) debug: - msg: "{{ docker_res.ansible_facts.docker_container.Output }}" - when: not DETACH|bool + msg: "{{ docker_res.ansible_facts.docker_container.Output[-20000:] }}" + when: not DETACH|bool and docker_res.ansible_facts is defined + - name: Get error + debug: + msg: "{{ docker_res.msg[-20000:] }}" + when: docker_res.ansible_facts is not defined + - name: Check status + debug: + msg: "Check container exit code" + failed_when: docker_res.ansible_facts is not defined or docker_res.ansible_facts.docker_container.State.ExitCode != 0 + diff --git a/org/ystia/docker/containers/generic/playbooks/stop.yaml b/org/ystia/docker/containers/generic/playbooks/stop.yaml index f8d6460a..fa3143e6 100644 --- a/org/ystia/docker/containers/generic/playbooks/stop.yaml +++ b/org/ystia/docker/containers/generic/playbooks/stop.yaml @@ -15,4 +15,10 @@ docker_container: name: "{{CONTAINER_ID}}" state: stopped + when: STARTED_CONTAINER_ID is not defined or STARTED_CONTAINER_ID == "" + - name: Stop started docker container + docker_container: + name: "{{STARTED_CONTAINER_ID}}" + state: stopped + when: STARTED_CONTAINER_ID is defined and STARTED_CONTAINER_ID != "" diff --git a/org/ystia/docker/containers/generic/types.yml b/org/ystia/docker/containers/generic/types.yml index 02a05039..0b1b0ed1 100644 --- a/org/ystia/docker/containers/generic/types.yml +++ b/org/ystia/docker/containers/generic/types.yml @@ -11,7 +11,7 @@ metadata: template_version: 3.0.0-SNAPSHOT template_author: Ystia - +description: Docker container types imports: - tosca-normative-types:1.0.0-ALIEN20 @@ -47,6 +47,10 @@ node_types: List of additional container ports which informs Docker that the container listens on the specified network ports at runtime. If the port is already exposed using EXPOSE in a Dockerfile, it does not need to be exposed again. required: false + hostname: + type: string + description: Container hostname + required: false image: type: string description: > @@ -59,6 +63,13 @@ node_types: description: > Retain volumes associated with a removed container. default: true + network_mode: + type: string + description: > + Connect the container to a network. + Possible values: bridge, none, host, | + or container: to reuse another container's network stack + required: false published_ports: type: list entry_schema: @@ -102,21 +113,26 @@ node_types: required: false attributes: container_id: { get_operation_output: [SELF, Standard, create, CONTAINER_ID] } + started_container_id: { get_operation_output: [SELF, Standard, create, STARTED_CONTAINER_ID] } interfaces: Standard: inputs: # Will be empty for create operation but not a big deal CONTAINER_ID: { get_attribute: [SELF, container_id] } + STARTED_CONTAINER_ID: { get_attribute: [SELF, started_container_id] } AUTO_REMOVE: { get_property: [SELF, auto_remove] } CLEANUP: { get_property: [SELF, cleanup] } + COMMAND: { get_property: [SELF, docker_run_cmd] } CPU_SHARE: { get_property: [SELF, cpu_share] } DETACH: { get_property: [SELF, detach] } ENV_VARS: { get_property: [SELF, docker_env_vars] } EXPOSED_PORTS: { get_property: [SELF, exposed_ports] } + HOSTNAME: { get_property: [SELF, hostname] } IMAGE: { get_property: [SELF, image] } KEEP_VOLUMES: { get_property: [SELF, keep_volumes] } MEM_SHARE: { get_property: [SELF, mem_share] } MEM_SHARE_LIMIT: { get_property: [SELF, mem_share_limit] } + NETWORK_MODE: { get_property: [SELF, network_mode] } PUBLISHED_PORTS: { get_property: [SELF, published_ports] } RESTART_POLICY: { get_property: [SELF, restart_policy] } SHM_SIZE: { get_property: [SELF, shm_size] } diff --git a/org/ystia/docker/images/playbooks/create.yaml b/org/ystia/docker/images/playbooks/create.yaml new file mode 100644 index 00000000..59c62aeb --- /dev/null +++ b/org/ystia/docker/images/playbooks/create.yaml @@ -0,0 +1,72 @@ +# +# Ystia Forge +# Copyright (C) 2018 Bull S. A. S. - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. +# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. +# +- name: Install ansible docker_image requirements + hosts: all + become: true + tasks: + - name: Get python version + python_requirements_info: + register: pri + failed_when: pri == None or pri.python_version == None or pri.python_version == '' + - name: Get python major version + set_fact: + python_major_version: "{{pri.python_version | replace('\n', '') | regex_replace('^(\\d+).*', '\\1') }}" + - name: Set python version for pip + set_fact: + python_pip_pkg: "python3-pip" + pip_cmd: "pip3" + when: python_major_version != "2" + - name: Set python 2 version for pip + set_fact: + python_pip_pkg: "python-pip" + pip_cmd: "pip" + when: python_major_version == "2" + - name: RedHat - install prerequisites + yum: + name: + - "{{ python_pip_pkg }}" + state: present + update_cache: yes + when: ansible_os_family == 'RedHat' + - name: Debian - install prerequisites + apt: + name: + - "{{ python_pip_pkg }}" + state: present + update_cache: yes + register: apt_res + retries: 5 + delay: 15 + until: apt_res is success + when: ansible_os_family == 'Debian' + - name: Install latest Pip version + pip: + name: "pip" + state: latest + executable: "{{pip_cmd}}" + when: python_major_version != "2" + - name: Install latest Pip version compatible with python 2 + pip: + name: "pip<21.0" + state: latest + executable: "{{pip_cmd}}" + when: python_major_version == "2" + - name: Add user to docker group + user: + name: "{{USER}}" + groups: docker + append: yes + when: USER != "root" + - name: Install 'docker' python package + pip: + name: docker + # In user directory to avoid potential issues with distutils + # see issue https://github.com/pypa/pip/issues/5247 + extra_args: --user + executable: "{{pip_cmd}}" + become_user: "{{USER}}" + become_method: sudo + diff --git a/org/ystia/docker/images/playbooks/load_archive.yaml b/org/ystia/docker/images/playbooks/load_archive.yaml new file mode 100644 index 00000000..590d0737 --- /dev/null +++ b/org/ystia/docker/images/playbooks/load_archive.yaml @@ -0,0 +1,40 @@ +# +# Ystia Forge +# Copyright (C) 2020 Bull S. A. S. - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. +# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. +# +- name: Load Docker image from tar archive + hosts: all + tasks: + - name: Get archive PATH property + set_fact: + archive_path: "{{PATH}}" + failed_when: PATH == "" + - name: Load tar archive + docker_image: + name: "{{NAME}}" + tag: "{{TAG}}" + push: "{{PUSH}}" + repository: "{{REPOSITORY}}" + load_path: "{{archive_path}}" + source: load + force_source: "{{FORCE_LOAD}}" + become: true + become_user: "{{USER}}" + become_method: sudo + register: result + - name: Fail if the image specified does not correpond to the archive specified + debug: + msg: "Checking if image {{NAME}} {{TAG}} is in tar archive {{PATH}}" + failed_when: result.image == None + - name: Check if an image was loaded + debug: + msg: "Image {{NAME}} {{TAG}} already loaded" + when: result.image == {} + - name: Intialize fact used in opration output + set_fact: + REPO_TAGS: [] + - name: Set repo tags output if returned by docker_image + set_fact: + REPO_TAGS: "{{result.image.RepoTags | to_json}}" + when: result.image != None and result.image != {} diff --git a/org/ystia/docker/images/playbooks/remove_image.yaml b/org/ystia/docker/images/playbooks/remove_image.yaml new file mode 100644 index 00000000..976cf03e --- /dev/null +++ b/org/ystia/docker/images/playbooks/remove_image.yaml @@ -0,0 +1,30 @@ +# +# Ystia Forge +# Copyright (C) 2020 Bull S. A. S. - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. +# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. +# +- name: Remove Docker image + hosts: all + become: true + tasks: + # Trusting first REPO_TAGS returned by docker_image + # if there is just one such image in the registry + - name: "Get image {{NAME}} in regsitry" + set_fact: + images: "{{REPO_TAGS | from_json}}" + - name: "Remove image {{NAME}} {{TAG}}" + docker_image: + name: "{{images[0]}}" + state: absent + when: images | length == 1 + become_user: "{{USER}}" + become_method: sudo + - name: "Remove image {{NAME}} {{TAG}}" + docker_image: + name: "{{NAME}}" + tag: "{{TAG}}" + repository: "{{REPOSITORY}}" + state: absent + when: images | length != 1 + become_user: "{{USER}}" + become_method: sudo diff --git a/org/ystia/docker/images/types.yml b/org/ystia/docker/images/types.yml new file mode 100644 index 00000000..168ae017 --- /dev/null +++ b/org/ystia/docker/images/types.yml @@ -0,0 +1,151 @@ +tosca_definitions_version: alien_dsl_2_0_0 + +# +# Ystia Forge +# Copyright (C) 2018 Bull S. A. S. - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. +# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. +# + +metadata: + template_name: org.ystia.docker.images + template_version: 3.0.0-SNAPSHOT + template_author: Ystia + +description: Docker image types + +imports: + - tosca-normative-types:1.0.0-ALIEN20 + - yorc-types:1.1.0 + +node_types: + org.ystia.docker.images.pub.ArchiveLoader: + derived_from: tosca.nodes.SoftwareComponent + abstract: true + description: Loads a docker image from a tar archive (abstract) + properties: + name: + type: string + description: > + Image name. Name format will be one of: name, repository/name, registry_server:port/name. + When pushing an image the name can optionally include the tag by appending ':tag_name'. + required: true + tag: + type: string + description: > + Tag added to the image when pushing. + If name property format is name:tag, then tag value from name will take precedence. + If not defined, docker considers the tag value is 'latest'. + required: false + repository: + type: string + description: > + Full path to a repository. Expects format repository:tag. + If no tag is provided, will use the value of the tag parameter or latest. + required: false + push: + type: boolean + description: Push the image to the registry. Specify the registry as part of the name or repository parameter. + default: false + required: false + force_load: + type: boolean + description: Load the archive even when the image already exists + default: false + required: false + user: + type: string + description: Perform the operation as this existing user + default: root + required: false + org.ystia.docker.images.ArchiveLoader: + derived_from: org.ystia.docker.images.pub.ArchiveLoader + description: Loads a docker image from a tar archive + properties: + path: + type: string + description: > + Path to the tar archive (can be compressed with gzip, bzip2, or xz)). + required: true + attributes: + # Array of repository name:tag in registry for the specified name. + # Empty if no name was specified or if the image was already loaded + # and force_load is set to false + repo_tags: { get_operation_output: [SELF, Standard, start, REPO_TAGS] } + interfaces: + Standard: + inputs: + PATH: { get_property: [SELF, path] } + USER: { get_property: [SELF, user] } + NAME: { get_property: [SELF, name] } + TAG: { get_property: [SELF, tag] } + REPOSITORY: { get_property: [SELF, repository] } + PUSH: { get_property: [SELF, push] } + # Install dependencies + create: + implementation: playbooks/create.yaml + # load the image from the tar archive + start: + inputs: + FORCE_LOAD: { get_property: [SELF, force_load] } + implementation: playbooks/load_archive.yaml + # Remove the image from docker registry + stop: + inputs: + REPO_TAGS: { get_attribute: [SELF, repo_tags] } + implementation: playbooks/remove_image.yaml + org.ystia.docker.images.RuntimePathArchiveLoader: + derived_from: org.ystia.docker.images.pub.ArchiveLoader + description: > + Loads a docker image from a tar archive whose path is provided at runtime + by a component implementing capability org.ystia.docker.images.relationships.ArchiveProvider + requirements: + - archive_provider: + capability: org.ystia.docker.images.capabilities.ArchiveProvider + relationship: org.ystia.docker.images.relationships.ArchiveProvider + occurrences: [1, 1] + attributes: + # Array of repository name:tag in registry for the specified name. + # Empty if no name was specified or if the image was already loaded + # and force_load is set to false + repo_tags: { get_operation_output: [SELF, Standard, start, REPO_TAGS] } + interfaces: + Standard: + inputs: + PATH: {get_attribute: [REQ_TARGET, archive_provider, path]} + USER: {get_attribute: [REQ_TARGET, archive_provider, user]} + NAME: { get_property: [SELF, name] } + TAG: { get_property: [SELF, tag] } + REPOSITORY: { get_property: [SELF, repository] } + PUSH: { get_property: [SELF, push] } + # Install dependencies + create: + implementation: playbooks/create.yaml + # load the image from the tar archive + start: + inputs: + FORCE_LOAD: { get_property: [SELF, force_load] } + implementation: playbooks/load_archive.yaml + # Remove the image from docker registry + stop: + inputs: + REPO_TAGS: { get_attribute: [SELF, repo_tags] } + implementation: playbooks/remove_image.yaml +capability_types: + org.ystia.docker.images.capabilities.ArchiveProvider: + derived_from: tosca.capabilities.Root + description: > + A capability provided by a component providing a docker image tar archive. + attributes: + path: + type: string + description: Path to the docker image tar archive + user: + type: string + description: User having access to this archive + default: root +relationship_types: + org.ystia.docker.images.relationships.ArchiveProvider: + derived_from: tosca.relationships.DependsOn + description: > + Relationship with a provider of docker image tar archive + valid_target_types: [ org.ystia.docker.images.capabilities.ArchiveProvider ] diff --git a/org/ystia/experimental/consul/linux/ansible/playbooks/consul_install.yaml b/org/ystia/experimental/consul/linux/ansible/playbooks/consul_install.yaml index ef773ccd..5523697a 100644 --- a/org/ystia/experimental/consul/linux/ansible/playbooks/consul_install.yaml +++ b/org/ystia/experimental/consul/linux/ansible/playbooks/consul_install.yaml @@ -65,16 +65,28 @@ register: pri failed_when: "pri == None or pri.python_version == None or pri.python_version == ''" + - name: Get python major version + set_fact: + python_major_version: "{{pri.python_version | replace('\n', '') | regex_replace('^(\\d+).*', '\\1') }}" + - name: Set python version for pip set_fact: python_pip_pkg: "python3-pip" pip_cmd: "pip3" - + when: python_major_version != "2" - name: Set python 2 version for pip set_fact: python_pip_pkg: "python-pip" pip_cmd: "pip" - when: pri.python_version is version('3', '<') + when: python_major_version == "2" + + - name: RedHat - install epel-release + yum: + name: + - epel-release + state: present + update_cache: yes + when: ansible_os_family == 'RedHat' - name: RedHat - install prerequisites yum: @@ -101,6 +113,13 @@ name: "pip" state: latest executable: "{{pip_cmd}}" + when: python_major_version != "2" + - name: Install latest Pip version compatible with python 2 + pip: + name: "pip<21.0" + state: latest + executable: "{{pip_cmd}}" + when: python_major_version == "2" - name: Copy python requirements copy: