Security enhancements (#98)

.ansible-lint

@@ -0,0 +1,18 @@
+var_naming_pattern: "^[a-zA-Z0-9_]*$"
+
+parseable: true
+
+exclude_paths:
+  - .git/
+  - .gitignore
+  - .cache/
+  - roles/istio
+
+skip_list:
+  - unnamed-task
+  - role-name
+  - var-naming
+
+warn_list:
+  - experimental
+  - no-changed-when

Dockerfile

@@ -7,7 +7,7 @@ RUN apt update && apt upgrade -y \
 
 FROM baseline as tool_builder
 ARG kustomize_version=3.7.0
-ARG kubectl_version=1.18.8
+ARG kubectl_version=1.19.9
 
 WORKDIR /build
 
@@ -22,6 +22,7 @@ ARG gcp_cli_version=334.0.0
 # Add extra packages
 RUN apt install -y gzip wget git git-lfs jq sshpass \
   && curl -s https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash \
+  && helm plugin install https://github.com/databus23/helm-diff \
   # AWS
   && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${aws_cli_version}.zip" -o "awscliv2.zip" \
   && unzip awscliv2.zip \
@@ -39,14 +40,15 @@ COPY --from=tool_builder /build/kustomize /usr/local/bin/kustomize
 WORKDIR /viya4-deployment/
 COPY . /viya4-deployment/
 
+ENV HOME=/viya4-deployment
+
 RUN pip install -r ./requirements.txt \
   && ansible-galaxy install -r ./requirements.yaml \
   && chmod -R g=u /etc/passwd /etc/group /viya4-deployment/ \
   && chmod 755 /viya4-deployment/docker-entrypoint.sh
 
 ENV PLAYBOOK=playbook.yaml
 ENV VIYA4_DEPLOYMENT_TOOLING=docker
-ENV HOME=/viya4-deployment
 ENV ANSIBLE_CONFIG=/viya4-deployment/ansible.cfg
 ENV PATH=$PATH:/google-cloud-sdk/bin/
 

README.md

@@ -44,8 +44,9 @@ Prior to running this playbook some infrastructure needs to be in place
 #### Kubernetes cluster
 
 You can either bring your own K8s cluster or use one of the Viya 4 IAC projects to create a cluster using terraform.
-  - [Viya 4 IaC for Azure](https://github.com/sassoftware/viya4-iac-azure)
   - [Viya 4 IaC for AWS](https://github.com/sassoftware/viya4-iac-aws)
+  - [Viya 4 IaC for Azure](https://github.com/sassoftware/viya4-iac-azure)
+  - [Viya 4 IaC for GCP](https://github.com/sassoftware/viya4-iac-gcp)
 
 
 #### Storage
@@ -246,7 +247,8 @@ See [troubleshooting](./docs/Troubleshooting.md) page.
 ## Additional Resources
 
 - [Viya Resource Guide](https://github.com/sassoftware/viya4-resource-guide)
-- [SAS Viya 4 Infrastructure as Code (IaC) for Microsoft Azure](https://github.com/sassoftware/viya4-iac-azure)
 - [SAS Viya 4 Infrastructure as Code (IaC) for Amazon Web Services (AWS)](https://github.com/sassoftware/viya4-iac-aws)
+- [SAS Viya 4 Infrastructure as Code (IaC) for Microsoft Azure](https://github.com/sassoftware/viya4-iac-azure)
+- [SAS Viya 4 Infrastructure as Code (IaC) for Google Cloud Platform (GCP)](https://github.com/sassoftware/viya4-iac-gcp)
 - [Viya Monitoring for Kubernetes](https://github.com/sassoftware/viya4-monitoring-kubernetes)
 - [Viya Orders CLI](https://github.com/sassoftware/viya4-orders-cli)

docs/CONFIG-VARS.md

@@ -162,6 +162,9 @@ When setting V4_CFG_MANAGE_STORAGE to true, A new storage classes will be create
 | V4M_KIBANA_CERT | Path to tls certificate to use for kibana ingress | string |<V4M_CERT> | false | If both this and V4M_CERT are not set a self-signed cert will be used | cluster-logging |
 | V4M_KIBANA_KEY | Path to tls key to use for kibana ingress | string | <V4M_KEY> | false | If both this and V4M_KEY are not set a self-signed cert will be used | cluster-logging |
 | V4M_KIBANA_PASSWORD | Kibana admin password | string | randomly generated | false | If not provided, a random password will be generated and written to the log output | cluster-logging |
+| V4M_KIBANASERVER_PASSWORD | Kibana server password | string | randomly generated | false | If not provided, a random password will be generated and written to the log output | cluster-logging |
+| V4M_LOGCOLLECTOR_PASSWORD | Logcollector password | string | randomly generated | false | If not provided, a random password will be generated and written to the log output | cluster-logging |
+| V4M_METRICGETTER_PASSWORD | Metricgetter password | string | randomly generated | false | If not provided, a random password will be generated and written to the log output | cluster-logging |
 | | | | | | | |
 | V4M_ELASTICSEARCH_FQDN | FQDN to use for elasticsearch ingress  | string | elasticsearch.<V4M_BASE_DOMAIN> | false | | cluster-logging |
 | V4M_ELASTICSEARCH_CERT | Path to tls certificate to use for elasticsearch ingress | string |<V4M_CERT> | false | If both this and V4M_CERT are not set a self-signed cert will be used | cluster-logging |

docs/user/Dependencies.md

@@ -13,15 +13,14 @@ SOURCE | NAME | VERSION
 ~ | docker | any
 ~ | git | any
 ~ | kustomize | 3.7.0
-~ | kubectl | 1.18.8
+~ | kubectl | 1.19.9
 ~ | AWS IAM Authenticator | 1.18.9/2020-11-02
 ~ | Helm | 3
-pip3 | ansible | 2.10.0
-pip3 | openshift | 0.11.2
-pip3 | kubernetes | 11.0.0
+pip3 | ansible | 2.10.7
+pip3 | openshift | 0.12.0
+pip3 | kubernetes | 12.0.1
 pip3 | dnspython | 2.1.0
-ansible-galaxy | community.kubernetes | 1.2.0
-ansible-galaxy | ansible.posix | 1.1.1
+ansible-galaxy | community.kubernetes | 1.2.1
 
 Required project dependencies are generally pinned to known working or stable versions to ensure users have a smooth initial experience. In some cases it may be required to change the default version of a dependency. In such cases users are welcome to experiment with alternate versions, however compatibility may not be guaranteed.
 

playbooks/playbook.yaml

@@ -11,6 +11,7 @@
     - name: common role
       include_role: 
         name: common
+        public: yes
       tags:
         - install
         - uninstall
@@ -19,34 +20,33 @@
       include_role:
         name: jump-server
       when:
-      - JUMP_SVR_HOST is defined
-      - JUMP_SVR_USER is defined
-      - JUMP_SVR_PRIVATE_KEY is defined
-      - V4_CFG_MANAGE_STORAGE is defined
-      - V4_CFG_MANAGE_STORAGE
+        - JUMP_SVR_HOST is defined
+        - JUMP_SVR_USER is defined
+        - JUMP_SVR_PRIVATE_KEY is defined
+        - V4_CFG_MANAGE_STORAGE is defined
+        - V4_CFG_MANAGE_STORAGE
       tags:
         - viya
     - name: baseline role
       include_role:
         name: baseline
       tags:
         - baseline
-    - include_vars:
-        file: "{{ DEPLOY_DIR }}/site-config/defaults.yaml"
-      when: CONFIG is not defined
+    - name: monitoring role - cluster
+      include_role:
+        name: monitoring
       tags:
-        - viya
+        - cluster-monitoring
+        - cluster-logging
     - name: vdm role
       include_role:
         name: vdm
       tags:
         - viya
-    - name: monitoring role
+    - name: monitoring role - namespace
       include_role:
         name: monitoring
       tags:
-        - cluster-monitoring
-        - cluster-logging
         - viya-monitoring
     - name: Delete tmpdir
       file:

requirements.txt

@@ -1,4 +1,4 @@
-ansible==2.10
-openshift==0.11.2
-kubernetes==11.0.0
-dnspython==2.1.0
+ansible==2.10.7
+openshift==0.12.0
+kubernetes==12.0.1
+dnspython==2.1.0

requirements.yaml

@@ -1,6 +1,4 @@
 ---
 collections:
   - name: community.kubernetes
-    version: 1.2.0
-  - name: ansible.posix
-    version: 1.1.1
+    version: 1.2.1

roles/istio/tasks/main.yml

@@ -1,13 +1,4 @@
 ---
-- name: Create tmp dir
-  tempfile:
-    state: directory
-  register: tmpdir
-  tags:
-    - install
-    - uninstall
-    - upgrade
-
 - name: Download istio
   shell: 
     cmd: "curl -L https://istio.io/downloadIstio | ISTIO_VERSION={{ istio_ver }} TARGET_ARCH={{ target_arch }} sh -"
@@ -70,7 +61,7 @@
         rules:
         - from:
           - source:
-              ipBlocks: "{{ loadBalancerSourceRanges }}"
+              ipBlocks: "{{ LOADBALANCER_SOURCE_RANGES }}"
   tags:
     - install
     - upgrade

roles/jump-server/tasks/main.yml

@@ -38,7 +38,7 @@
 - name: jump-server - create folders
   file:
     state: directory
-    path: "{{ JUMP_SVR_RWX_FILESTORE_PATH | replace('/$', '') }}/{{ hostvars['localhost']['NAMESPACE']}}/{{ item }}"
+    path: "{{ JUMP_SVR_RWX_FILESTORE_PATH | replace('/$', '') }}/{{ hostvars['localhost']['NAMESPACE'] }}/{{ item }}"
     owner: "{{ folder_owner }}"
     group: "{{ folder_group }}"
     mode: "0777"
@@ -50,4 +50,4 @@
   delegate_to: "{{ groups['jump'][0] }}"
   become: yes
   tags:
-    - install
+    - install

roles/monitoring/defaults/main.yaml

@@ -12,7 +12,10 @@ V4M_KEY: null
 V4M_KIBANA_FQDN: "kibana.{{ V4M_BASE_DOMAIN }}"
 V4M_KIBANA_CERT: "{{ V4M_CERT }}"
 V4M_KIBANA_KEY: "{{ V4M_KEY }}"
-V4M_KIBANA_PASSWORD: null
+V4M_KIBANA_PASSWORD: "{{ lookup('password', '/dev/null chars=ascii_letters,digits') }}"
+V4M_KIBANASERVER_PASSWORD: "{{ lookup('password', '/dev/null chars=ascii_letters,digits') }}"
+V4M_LOGCOLLECTOR_PASSWORD: "{{ lookup('password', '/dev/null chars=ascii_letters,digits') }}"
+V4M_METRICGETTER_PASSWORD: "{{ lookup('password', '/dev/null chars=ascii_letters,digits') }}"
 
 V4M_ELASTICSEARCH_FQDN: "elasticsearch.{{ V4M_BASE_DOMAIN }}"
 V4M_ELASTICSEARCH_CERT: "{{ V4M_CERT }}"
@@ -29,4 +32,4 @@ V4M_ALERTMANAGER_KEY: "{{ V4M_KEY }}"
 V4M_GRAFANA_FQDN: "grafana.{{ V4M_BASE_DOMAIN }}"
 V4M_GRAFANA_CERT: "{{ V4M_CERT }}"
 V4M_GRAFANA_KEY: "{{ V4M_KEY }}"
-V4M_GRAFANA_PASSWORD: null
+V4M_GRAFANA_PASSWORD: "{{ lookup('password', '/dev/null chars=ascii_letters,digits') }}"

roles/monitoring/tasks/cluster-logging.yaml

@@ -3,75 +3,67 @@
   file:
     path: "{{ tmpdir.path }}/logging/"
     state: directory
+    mode: "0770"
   tags:
     - install
     - uninstall
     - upgrade
 
-- name: cluster-logging - lookup creds
+- name: cluster-logging - lookup existing credentials
   community.kubernetes.k8s_info:
     api_version: v1
     kind: Secret
-    name: internal-user-admin
-    namespace: logging
+    namespace: "logging"
     kubeconfig: "{{ KUBECONFIG }}"
-  register: logging_creds
+    label_selectors:
+      - managed-by = v4m-es-script
+  register: "logging_secrets"
   tags:
     - install
-    - upgrade
-    - uninstall
 
-- set_fact:
-    V4M_KIBANA_PASSWORD: "{{logging_creds.resources[0].data.password|b64decode}}"
-  tags:
-    - install
-    - upgrade
-    - uninstall
+- name: cluster-logging - save credentials
+  set_fact: 
+    "{{ logging_map['secret'][item.metadata.name] }}": "{{ item.data.password|b64decode }}"
+  with_items: "{{ logging_secrets.resources }}"
   when:
-    - (logging_creds.resources | length) == 1
-
-- name: cluster-logging - generate kibana password
-  set_fact:
-    V4M_KIBANA_PASSWORD: "{{ lookup('password', '/dev/null chars=ascii_letters,digits') }}"
+    - item.metadata is defined
+    - item.metadata.name is defined
+    - item.metadata.name in ("internal-user-admin", "internal-user-kibanaserver", "internal-user-logcollector", "internal-user-metricgetter")
   tags:
     - install
-  when:
-    - V4M_KIBANA_PASSWORD is none
 
-- name: cluster-logging - credentials
+- name: cluster-logging - output credentials
   debug:
     msg:
-      - "Kibana username: admin" 
-      - "Kibana password: {{ V4M_KIBANA_PASSWORD }}"
+      - "Kibana admin  - username: admin,        password: {{ V4M_KIBANA_PASSWORD }}"
+      - "Kibana Server - username: kibanaserver, password: {{ V4M_KIBANASERVER_PASSWORD }}"
+      - "Log Collector - username: logcollector, password: {{ V4M_LOGCOLLECTOR_PASSWORD }}"
+      - "Metric Getter - username: metricgetter, password: {{ V4M_METRICGETTER_PASSWORD }}"
   tags:
     - install
 
 - name: cluster-logging - user values
   template:
     src: "user-values-elasticsearch-open.yaml"
     dest: "{{ tmpdir.path }}/logging/user-values-elasticsearch-open.yaml"
+    mode: "0660"
   tags:
     - install
     - update
     - uninstall
 
 - name: cluster-logging - deploy
-  shell:
+  command:
     cmd: "{{ tmpdir.path }}/viya4-monitoring-kubernetes/logging/bin/deploy_logging_open.sh"
-  environment:
-    USER_DIR: "{{ tmpdir.path }}"
-    TLS_ENABLE: "true"
-    LOG_KB_TLS_ENABLE: "true"
-    KUBECONFIG: "{{ KUBECONFIG }}"
-    LOG_COLOR_ENABLE: false
-    NODE_PLACEMENT_ENABLE: "{{ V4M_NODE_PLACEMENT_ENABLE }}"
-    ES_ADMIN_PASSWD: "{{ V4M_KIBANA_PASSWORD }}"
-  ignore_errors: yes
+  environment: "{{ logging_map['env'] }}"
+  register: result
+  failed_when:
+    - "'can be ignored' not in result.stdout"
   tags:
     - install
     - update
 
-- name: cluster-monitoring - elasticsearch cert
+- name: cluster-logging - elasticsearch cert
   community.kubernetes.k8s:
     kubeconfig: "{{ KUBECONFIG }}"
     state: present
@@ -94,7 +86,7 @@
     - install
     - update
 
-- name: cluster-monitoring - kibana cert
+- name: cluster-logging - kibana cert
   community.kubernetes.k8s:
     kubeconfig: "{{ KUBECONFIG }}"
     state: present
@@ -118,12 +110,19 @@
     - update
 
 - name: cluster-logging - uninstall
-  shell:
+  command:
     cmd: "{{ tmpdir.path }}/viya4-monitoring-kubernetes/logging/bin/remove_logging_open.sh"
-  environment:
-    USER_DIR: "{{ tmpdir.path }}"
-    TLS_ENABLE: "true"
-    KUBECONFIG: "{{ KUBECONFIG }}"
-    LOG_COLOR_ENABLE: false
+  environment: "{{ logging_map['env'] }}"
+  tags:
+    - uninstall
+
+- name: cluster-logging - delete namespace
+  community.kubernetes.k8s:
+    api_version: v1
+    kind: Namespace
+    name: "logging"
+    wait: true
+    kubeconfig: "{{ KUBECONFIG }}"
+    state: absent
   tags:
     - uninstall