Release v1.2.2 (#37)

* Run scripts should now ignore errors, and hopefully produce files as intended

* Artifacts should now be created in the running directory

* Creating individual artifact directories and doing result scan after

* Fixed a ws issue with run_scripts playbook

* Run scripts did not accept the chdir arg

* Fixed typo for scripts results variable

* Adding archive step to run tests since it should really be part of the default test

* Explicitly calling archive since vars my not be referencable from other vars

* Linchpin workspace should now default to libraries spec

* Updated tests and fixed codenarc issues

* Update release version and add parsing function

* Codenarc narcing all over the new function

* Updating CI Message to have better error handling and a map return

* SSH mode support and remote target host update

* Added lines to cover installing cert for RCM tools

* Installing certs on host so we can pull down dev imgs

* Ensuring backup cert install is still available

* Fixed curl requests so that they will install certs correctly

* Credentials should now be installed correctly on SSH mode hosts, and guaranteed install when rhpkg is specified

* Moved script object into the final closure, so objects can be passed to it in SSH mode.

* Updated logic to allow for non-root ssh mode, secret file injection, and env exports

* Inject secret locations failed, so attemting to remove layer of indirection

* Updating utils to once against attempt variable injection

* Codenarc is a narc

* Attempting to escape $ so that file in inject

* Updated local install to mkdirs before copying files there

* Removed Jenkins krb principal for files list

* We should now install the correct rhpkg tools per the rhel host spec

* Added overridable map for inventory vars

* Fixed syntax for NoOpProvisioner inventory vars override

* Renamed inventoryVars, updated tests, and ensured proper functioning for NoOpProvisioner

* Variable naming convention consistency in LinchPin workspace

* Reading hostname and ensuring only the hostname is picked up

* Package name fix for RHEL-8

* Renamed resource playbooks to indicate what they were

* Fixed distribution version

* Fixed ternary grouping

* Fixing python package install on RHEL 8

* Fixed path issue for renamed playbooks

* RHEL 7 targets should no longer attempt to install python 3

* Renamed test output file to runlog

* CI message should now be cleaned up

* Pulled script var back out into the genericInstall since JNLP mode needs to be able to change the node BEFORE calling the sh step

resources/playbooks/run_scripts.yml

@@ -0,0 +1,64 @@
+---
+- name: "Run Scripts on each Inventory Host"
+  hosts: all
+  gather_facts: true
+  vars_prompt:
+    - name: test_dir
+      prompt: "Please enter the path to your tests"
+
+  tasks:
+   - name: install rsync and the utilities to copy the tests to the remote host
+     package:
+       name: "{{ item }}"
+       state: latest
+     with_items:
+       - rsync
+       - "{{ ((ansible_facts['distribution'] == 'RedHat' or ansible_facts['distribution'] == 'CentOS') and 
+              ansible_facts['distribution_major_version'] | int >= 8) | 
+                 ternary('python3-libselinux', 'libselinux-python') }}"
+       - "{{ ((ansible_facts['distribution'] == 'RedHat' or ansible_facts['distribution'] == 'CentOS') and 
+              ansible_facts['distribution_major_version'] | int >= 8) | 
+                 ternary('python3', 'python2') }}"
+
+   - name: create a directory for the tests on the remote host
+     file:
+       path: "/tmp/{{ test_dir }}"
+       state: directory
+
+   - name: copy the tests onto the hosts where they will be executed
+     synchronize:
+       src: "{{ test_dir }}/scripts"
+       dest: "/tmp/{{ test_dir }}"
+       mode: push
+     ignore_errors: true
+
+   - name: scan and find the test scripts directory
+     find:
+       recurse: false
+       paths: "/tmp/{{ test_dir }}/scripts"
+       file_type: directory
+     register: scripts_dir
+
+   - name: loop through script directories and create artifacts directory
+     file:
+       path: "{{ script_dir }}/artifacts"
+       state: directory
+     loop: "{{ scripts_dir.files | map(attribute='path') | list }}"
+     loop_control:
+        loop_var: script_dir
+
+   - name: loop through script directories, run test, and store artifacts
+     shell: "bash -x test.sh {{ script_args | default('') }} &> artifacts/{{ ansible_architecture }}-runlog.txt"
+     args:
+       chdir: "{{ script_dir }}"
+     loop: "{{ scripts_dir.files | map(attribute='path') | list }}"
+     loop_control:
+        loop_var: script_dir
+     ignore_errors: yes
+     register: scripts
+
+   - name: fail if the return code is not 0
+     fail:
+       msg: "The command ({{ item.cmd }}) failed with return code {{ item.rc }}"
+     when: item.rc != 0
+     with_items: "{{ scripts.results }}"

src/com/redhat/ci/Utils.groovy

@@ -12,6 +12,17 @@ class Utils {
     private static final String SUDO = 'sudo '
     private static final String NO_SUDO = ''
     private static final String INSTALL_FILE = 'install.sh'
+    private static final String SSH_ARGS = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+    private static final String SSH_IDENTITY = '-i ~/.ssh/id_rsa'
+    private static final String SSH = "ssh ${SSH_ARGS} ${SSH_IDENTITY}"
+    private static final String SCP = "scp ${SSH_ARGS} ${SSH_IDENTITY}"
+    private static final String HOME = '/home/jenkins'
+    private static final String KEYTAB = 'KEYTAB'
+    private static final String KRB_PRINCIPAL = 'KRB_PRINCIPAL'
+    private static final String SSHPRIVKEY = 'SSHPRIVKEY'
+    private static final String SSHPUBKEY = 'SSHPUBKEY'
+    private static final String KRBCONF = 'KRBCONF'
+    private static final String BKRCONF = 'BKRCONF'
 
     /**
      * Attemps to install Ansible.
@@ -28,7 +39,7 @@ class Utils {
                 ${sudo}pip install --upgrade pip &&
                 ${sudo}pip install --upgrade setuptools &&
                 ${sudo}pip install --upgrade ansible
-            """)
+            """, null)
             if (host == null) {
                 return
             }
@@ -44,16 +55,22 @@ class Utils {
             privileged, sh ->
             String sudo = privileged ? SUDO : NO_SUDO
             script.withCredentials([
-                script.file(credentialsId:config.keytabCredentialId, variable:'KEYTAB'),
+                script.file(credentialsId:config.keytabCredentialId, variable:KEYTAB),
                 script.usernamePassword(credentialsId:config.krbPrincipalCredentialId,
-                                        usernameVariable:'KRB_PRINCIPAL',
+                                        usernameVariable:KRB_PRINCIPAL,
                                         passwordVariable:''),
-                script.file(credentialsId:config.sshPrivKeyCredentialId, variable:'SSHPRIVKEY'),
-                script.file(credentialsId:config.sshPubKeyCredentialId,  variable:'SSHPUBKEY'),
-                script.file(credentialsId:config.krbConfCredentialId,    variable:'KRBCONF'),
-                script.file(credentialsId:config.bkrConfCredentialId,    variable:'BKRCONF'),
+                script.file(credentialsId:config.sshPrivKeyCredentialId, variable:SSHPRIVKEY),
+                script.file(credentialsId:config.sshPubKeyCredentialId,  variable:SSHPUBKEY),
+                script.file(credentialsId:config.krbConfCredentialId,    variable:KRBCONF),
+                script.file(credentialsId:config.bkrConfCredentialId,    variable:BKRCONF),
             ]) {
-                script.env.HOME = '/home/jenkins'
+                script.env.HOME = HOME
+                Map context = [
+                    env:[HOME:HOME],
+                    files:[script.KEYTAB, script.SSHPRIVKEY,
+                           script.SSHPUBKEY, script.KRBCONF, script.BKRCONF],
+                ]
+
                 sh("""
                     ${sudo}yum install -y krb5-workstation
                     ${sudo}cp ${script.KRBCONF} /etc/krb5.conf
@@ -69,7 +86,7 @@ class Utils {
                     chmod 644 ~/.ssh/id_rsa.pub
                     eval "\$(ssh-agent -s)"
                     ssh-add ~/.ssh/id_rsa
-                """)
+                """, context)
                 if (host != null) {
                     host.credentialsInstalled = true
                 }
@@ -85,18 +102,35 @@ class Utils {
         genericInstall(script, config, host) {
             privileged, sh ->
             String sudo = privileged ? SUDO : NO_SUDO
+            String distro = host ? host.distro : 'RHEL-7'
+            String variant = host ? host.variant : 'Server'
+            if (!distro || !variant || !distro.startsWith('RHEL')) {
+                script.echo("Installing rhpkg tool is not supported for distro=[${distro}] and variant=[${variant}]")
+                return
+            }
+            String osMajorVersion = distro.find('[0-9]+')
+            if (!osMajorVersion || osMajorVersion.toInteger() < 5 || osMajorVersion.toInteger() > 8) {
+                script.echo("RCM rhpkg tool is not available for distro=[${distro}]. Invalid major version=[${osMajorVersion}]")
+                return
+            }
             sh("""
                 echo "pkgs.devel.redhat.com,10.19.208.80 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAplqWKs26qsoaTxvWn3DFcdbiBxqRLhFngGiMYhbudnAj4li9/VwAJqLm1M6YfjOoJrj9dlmuXhNzkSzvyoQODaRgsjCG5FaRjuN8CSM/y+glgCYsWX1HFZSnAasLDuW0ifNLPR2RBkmWx61QKq+TxFDjASBbBywtupJcCsA5ktkjLILS+1eWndPJeSUJiOtzhoN8KIigkYveHSetnxauxv1abqwQTk5PmxRgRt20kZEFSRqZOJUlcl85sZYzNC/G7mneptJtHlcNrPgImuOdus5CW+7W49Z/1xqqWI/iRjwipgEMGusPMlSzdxDX4JzIx6R53pDpAwSAQVGDz4F9eQ==" | ${sudo}tee -a /etc/ssh/ssh_known_hosts
 
                 echo "Host pkgs.devel.redhat.com" | ${sudo}tee -a /etc/ssh/ssh_config
                 echo "IdentityFile /home/jenkins/.ssh/id_rsa" | ${sudo}tee -a /etc/ssh/ssh_config
 
+                ${sudo}curl -o /etc/pki/ca-trust/source/anchors/RedHat_CA.crt -k -L https://password.corp.redhat.com/cacert.crt
+                ${sudo}curl -o /etc/pki/ca-trust/source/anchors/PnTDevOps_CA.crt -k -L https://engineering.redhat.com/Eng-CA.crt
+                ${sudo}curl -o /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt -k -L https://password.corp.redhat.com/RH-IT-Root-CA.crt
+                ${sudo}update-ca-trust extract
+
                 ${sudo}yum install -y yum-utils git
-                curl -L -O http://download.devel.redhat.com/rel-eng/internal/rcm-tools-rhel-7-server.repo
-                ${sudo}yum-config-manager --add-repo rcm-tools-rhel-7-server.repo
+
+                curl -L -o rcm-tools.repo http://download.devel.redhat.com/rel-eng/internal/rcm-tools-rhel-${osMajorVersion}-${variant.toLowerCase()}.repo
+                ${sudo}yum-config-manager --add-repo rcm-tools.repo
                 ${sudo}yum install -y rhpkg
                 git config --global user.name "jenkins"
-            """)
+            """, null)
             if (host != null) {
                 host.rhpkgInstalled = true
             }
@@ -112,7 +146,7 @@ class Utils {
         // Installation should occur on current node
         if (host == null) {
             installWrapper(NO_SUDO) {
-                shCommand ->
+                shCommand, context=[:] ->
                 script.sh(shCommand)
             }
             return
@@ -126,7 +160,7 @@ class Utils {
 
             script.node(host.displayName) {
                 installWrapper(SUDO) {
-                    shCommand ->
+                    shCommand, context=[:] ->
                     script.sh(shCommand)
                 }
             }
@@ -139,12 +173,31 @@ class Utils {
                 throw new ProvisioningException('Installing in SSH mode but hostname is invalid.')
             }
 
-            installWrapper(NO_SUDO) {
-                shCommand ->
-                script.writeFile(file:INSTALL_FILE, text:shCommand)
-                String runCommandOnHost = 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ' +
-                    "-i ~/.ssh/id_rsa root@${host.hostname} < ${INSTALL_FILE}"
-                script.sh(runCommandOnHost)
+            installWrapper(host.remoteUser == 'root' ? NO_SUDO : SUDO) {
+                shCommand, context=[:] ->
+                // Copy files onto target host
+                String files = ''
+                if (context && context.files) {
+                    for (file in context.files) {
+                        files += """
+                            $SSH ${host.remoteUser}@${host.hostname} mkdir -p \$(dirname "$file") &&
+                            $SCP $file ${host.remoteUser}@${host.hostname}:$file;\n
+                        """
+                    }
+                }
+                script.sh(files)
+
+                // Export env vars
+                String exports = ''
+                if (context && context.env) {
+                    for (envVar in context.env) {
+                        exports += "export ${envVar.key}=${envVar.value};\n"
+                    }
+                }
+
+                // Run the actual script
+                script.writeFile(file:INSTALL_FILE, text:"$exports $shCommand")
+                script.sh("$SSH ${host.remoteUser}@${host.hostname} < $INSTALL_FILE")
             }
         }
     }

src/com/redhat/ci/hosts/ProvisionedHost.groovy

@@ -56,5 +56,7 @@ class ProvisionedHost extends TargetHost {
         this.bkrMethod = target.bkrMethod
         this.reserveDuration = target.reserveDuration
         this.scriptParams = target.scriptParams
+        this.remoteUser = target.remoteUser
+        this.inventoryVars = target.inventoryVars
     }
 }

src/com/redhat/ci/hosts/TargetHost.groovy

@@ -38,4 +38,10 @@ class TargetHost extends Host {
 
     // String of parameters to pass to script tests
     String scriptParams = null
+
+    // Remote user to connect with in SSH mode
+    String remoteUser = 'root'
+
+    // Inventory file variables
+    Map inventoryVars = [:]
 }

src/com/redhat/ci/provisioner/ProvisioningConfig.groovy

@@ -4,16 +4,20 @@ package com.redhat.ci.provisioner
  * Configuration needed to provision resources with a Provisioner.
  */
 class ProvisioningConfig {
+    private static final String RELEASE_VERSION = 'v1.2.2'
     private static final String KRB_PRINCIPAL_CREDENTIAL_ID_DEFAULT = 'redhat-multiarch-qe-krbprincipal'
     private static final String KEYTAB_CREDENTIAL_ID_DEFAULT = 'redhat-multiarch-qe-keytab'
     private static final String SSH_PRIV_KEY_CREDENTIAL_ID_DEFAULT = 'redhat-multiarch-qe-sshprivkey'
     private static final String SSH_PUB_KEY_CREDENTIAL_ID_DEFAULT = 'redhat-multiarch-qe-sshpubkey'
     private static final String JENKINS_SLAVE_CREDENTIAL_ID_DEFAULT = 'jenkins-slave-credentials'
     private static final String JENKINS_MASTER_URL_DEFAULT = ''
     private static final String JSWARM_EXTRA_ARGS_DEFAULT  = ''
+    private static final String PROVISIONING_REPO_URL_DEFAULT =
+      'https://github.com/redhat-multiarch-qe/multiarch-ci-libraries'
+    private static final String PROVISIONING_REPO_REF_DEFAULT = RELEASE_VERSION
 
     // Provisioner version
-    String version = 'v1.2.1'
+    String version = RELEASE_VERSION
 
     // Jenkins kubernetes cloud name
     String cloudName = 'openshift'
@@ -34,10 +38,10 @@ class ProvisioningConfig {
     String provisioningImage = 'provisioner'
 
     // Provisioning repo url
-    String provisioningRepoUrl = 'https://github.com/redhat-multiarch-qe/multiarch-ci-libraries'
+    String provisioningRepoUrl = PROVISIONING_REPO_URL_DEFAULT
 
     // Provisioning repo ref
-    String provisioningRepoRef = this.version
+    String provisioningRepoRef = PROVISIONING_REPO_REF_DEFAULT
 
     // Provisioning workspace location (needed for LinchPin)
     // This can reference a relative path in the above repo
@@ -116,6 +120,8 @@ class ProvisioningConfig {
         this.jenkinsSlaveCredentialId = params.JENKINSSLAVECREDENTIALID ?: this.jenkinsSlaveCredentialId
         this.jenkinsMasterUrl = env.JENKINS_MASTER_URL ?: this.jenkinsMasterUrl
         this.jswarmExtraArgs = env.JSWARM_EXTRA_ARGS ?: this.jswarmExtraArgs
+        this.provisioningRepoUrl = params.LIBRARIES_REPO ?: this.provisioningRepoUrl
+        this.provisioningRepoRef = params.LIBRARIES_REF ?: this.provisioningRepoRef
 
         hostTypePriority = [
             com.redhat.ci.host.Type.CONTAINER,

src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy

@@ -114,12 +114,11 @@ class LinchPinProvisioner extends AbstractProvisioner {
                 if (config.installAnsible) {
                     Utils.installAnsible(script, config, host)
                 }
+            }
 
-                // In JNLP mode, install provisioning credentials directly on the provisioned host
-                // (Already installed in SSH mode)
-                if (config.installCredentials) {
-                    Utils.installCredentials(script, config, host)
-                }
+            // Install credentials directly on the provisioned host
+            if (config.installCredentials || config.installRhpkg) {
+                Utils.installCredentials(script, config, host)
             }
 
             // We can install the RHPKG tool if the user intends to use it.
@@ -198,6 +197,7 @@ class LinchPinProvisioner extends AbstractProvisioner {
             reserve_duration:host.reserveDuration,
             job_group:host.bkrJobGroup ?: config.jobgroup,
             hostrequires:getHostRequires(host, config),
+            inventory_vars:host.inventoryVars,
         ]
 
         JsonOutput.toJson(templateData)
@@ -252,7 +252,7 @@ class LinchPinProvisioner extends AbstractProvisioner {
     }
 
     private String getHostname(ProvisionedHost host) {
-        String getMasterNode = "awk '/\\[master_node\\]/{getline; print}' ${host.inventoryPath}"
+        String getMasterNode = "awk '/\\[master_node\\]/{getline; print}' ${host.inventoryPath} | cut -d ' ' -f 1"
         script.sh(returnStdout:true, script:getMasterNode).trim()
     }
 }

src/com/redhat/ci/provisioners/NoOpProvisioner.groovy

@@ -91,12 +91,11 @@ class NoOpProvisioner extends AbstractProvisioner {
                 if (config.installAnsible) {
                     Utils.installAnsible(script, config, host)
                 }
+            }
 
-                // In JNLP mode, install provisioning credentials directly on the provisioned host
-                // (Already installed in SSH mode)
-                if (config.installCredentials) {
-                    Utils.installCredentials(script, config, host)
-                }
+            // Install credentials directly on the provisioned host
+            if (config.installCredentials || config.installRhpkg) {
+                Utils.installCredentials(script, config, host)
             }
 
             // We can install the RHPKG tool if the user intends to use it.
@@ -173,7 +172,12 @@ class NoOpProvisioner extends AbstractProvisioner {
 
             // Build a cinch-compatible inventory using the passing in hostname
             for (String group in LAYOUT_GROUPS) {
-                inventory += "[${group}]\n${host.hostname}\n\n"
+                inventory += "[${group}]\n${host.hostname}"
+                host.inventoryVars.each {
+                    key, value ->
+                    inventory += " ${key}=${value}"
+                }
+                inventory += '\n\n'
             }
 
             // Write and return inventory file path