Skip to content

Commit

Permalink
Merge pull request #3998 from zowe/user/muzikovam/zoweYamlDiff
Browse files Browse the repository at this point in the history
User/muzikovam/zowe yaml diff
  • Loading branch information
MarkAckert authored Sep 20, 2024
2 parents 5508691 + 3f3eafb commit c88c59a
Show file tree
Hide file tree
Showing 16 changed files with 371 additions and 140 deletions.
10 changes: 9 additions & 1 deletion .github/workflows/build-packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -353,8 +353,8 @@ jobs:
jfrog rt dl ${{github.event.inputs.PSWI_SMPE_AZWE_ARTIFACTORY_PATH}}/AZWE002*.zip --flat=true .pax/AZWE002.zip
jfrog rt dl ${{github.event.inputs.PSWI_SMPE_ARTIFACTORY_PATH}}/zowe-smpe-*.zip --flat=true .pax/zowe-smpe.zip
- name: '[SMPE Pax 4] Build PSWI'
id: pswi
if: env.INPUTS_BUILD_PSWI == 'true'
timeout-minutes: 60
run: |
Expand All @@ -367,6 +367,14 @@ jobs:
ZZOW_SSH_PORT: ${{ secrets.SSH_MARIST_ALLSYS_PORT }}
VERSION: ${{ env.P_VERSION }}

- name: Store PSWI folder
uses: actions/upload-artifact@v4
if: env.INPUTS_BUILD_PSWI == 'true' && failure()
with:
name: pswi-folder
path: |
pswi/**
- name: '[K8S] Build Kubernetes'
timeout-minutes: 10
if: env.INPUTS_BUILD_KUBERNETES == 'true'
Expand Down
52 changes: 52 additions & 0 deletions .github/workflows/pswi-zowe-config-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
name: PSWI zowe.yaml sync check

permissions:
id-token: write
issues: write
pull-requests: write
contents: write

on:
push:
branches:
- v3.x/staging
pull_request:
types: [opened, synchronize]
workflow_dispatch:

jobs:
test-pswi-zowe-yaml:
runs-on: ubuntu-latest
steps:

- name: '[Prep 1] Checkout'
uses: actions/checkout@v4

- name: '[PSI-LOCK] Lock marist servers to build PSWI'
uses: zowe-actions/shared-actions/lock-resource@main
with:
lock-repository: ${{ github.repository }}
github-token: ${{ secrets.GITHUB_TOKEN }}
lock-resource-name: zowe-psi-build-zzow07-lock
lock-avg-retry-interval: 30

- name: 'Test ZWECONF workflow'
id: zweconf
timeout-minutes: 30
run: |
cd pswi
./ZWECONF_test.sh
env:
ZOSMF_USER: ${{ secrets.ZOWE_PSWI_BUILD_USR }}
ZOSMF_PASS: ${{ secrets.ZOWE_PSWI_BUILD_PASSWD }}
ZZOW_SSH_PORT: ${{ secrets.SSH_MARIST_ALLSYS_PORT }}

- name: Store test output
uses: actions/upload-artifact@v4
if: steps.zweconf.outcome != '' && failure()
with:
name: zweconf_test
path: |
pswi/**.txt
pswi/zowe_.yaml
pswi/ZWECONF1/**
4 changes: 2 additions & 2 deletions pswi/01_smpe.sh
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,8 @@ STATUS_NAME=`echo $RESP | grep -o '"statusName":".*"' | cut -f4 -d\"`

if [ "$STATUS_NAME" = "in-progress" ]
then
echo "Workflow ended with an error."
echo $RESP
echo "Workflow with SMP/E ended with an error." >> report.txt
echo $RESP >> report.txt
exit -1
elif [ "$STATUS_NAME" = "complete" ]
then
Expand Down
4 changes: 2 additions & 2 deletions pswi/02_ptf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ STATUS_NAME=`echo $RESP | grep -o '"statusName":".*"' | cut -f4 -d\"`

if [ "$STATUS_NAME" = "in-progress" ]
then
echo "Workflow ended with an error."
echo $RESP
echo "Workflow with PTFs ended with an error." >> report.txt
echo $RESP >> report.txt
exit -1
elif [ "$STATUS_NAME" = "complete" ]
then
Expand Down
20 changes: 10 additions & 10 deletions pswi/03_create.sh
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@ then
then
echo "${ZOWE_MOUNT} with zFS ${ZOWE_ZFS} mounted will be used."
else
echo "The file system ${ZOWE_ZFS} exists but is mounted to different mount point ${MOUNTZ}."
echo "It is required to have the file system ${ZOWE_ZFS} mounted to the exact mount point (${ZOWE_MOUNT}) to successfully export Zowe PSWI."
echo "The file system ${ZOWE_ZFS} exists but is mounted to different mount point ${MOUNTZ}." >> report.txt
echo "It is required to have the file system ${ZOWE_ZFS} mounted to the exact mount point (${ZOWE_MOUNT}) to successfully export Zowe PSWI." >> report.txt
exit -1
fi
else
Expand All @@ -90,7 +90,7 @@ else
if [ -n "$MOUNTZFS" ]
then
# If ZFS is not mounted to the mountpoint then this ZOWE mountpoint has different zFS
echo "The mountpoint ${ZOWE_MOUNT} has different zFS ${MOUNTZFS}."
echo "The mountpoint ${ZOWE_MOUNT} has different zFS ${MOUNTZFS}." >> report.txt
exit -1
else
# Mount zFS to Zowe mountpoint
Expand All @@ -108,14 +108,14 @@ RESP=`curl -s $CHECK_WORKFLOW_DSN_URL -k -X "GET" -H "Content-Type: application/
DS_COUNT=`echo $RESP | grep -o '"returnedRows":[0-9]*' | cut -f2 -d:`
if [ $DS_COUNT -ne 0 ]
then
echo "The ${WORKFLOW_DSN} already exist. Because there is a possibility that it contains something unwanted the script does not continue."
echo "The ${WORKFLOW_DSN} already exist. Because there is a possibility that it contains something unwanted the script does not continue." >> report.txt
exit -1
else
echo "Creating a data set where the post-Deployment workflow will be stored."
RESP=`curl -s $WORKFLOW_DSN_URL -k -X "POST" -d "$ADD_WORKFLOW_DSN_JSON" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
if [ -n "$RESP" ]
then
echo "The creation of the ${WORKFLOW_DSN} was not successful. Error message: ${RESP}"
echo "The creation of the ${WORKFLOW_DSN} was not successful. Error message: ${RESP}" >> report.txt
exit -1
fi
fi
Expand Down Expand Up @@ -173,7 +173,7 @@ RESP=`curl -s $CHECK_EXPORT_DSN_URL -k -X "GET" -H "Content-Type: application/js
DSN_COUNT=`echo $RESP | grep -o '"returnedRows":[0-9]*' | cut -f2 -d:`
if [ $DSN_COUNT -ne 0 ]
then
echo "The ${EXPORT_DSN} already exist. Because there is a possibility that it contains something unwanted the script does not continue."
echo "The ${EXPORT_DSN} already exist. Because there is a possibility that it contains something unwanted the script does not continue." >> report.txt
exit -1
else
echo "Creating a data set where the export jobs will be stored."
Expand Down Expand Up @@ -209,7 +209,7 @@ if [ $? -gt 0 ];then exit -1;fi
LOAD_STATUS_URL=`echo $RESP | grep -o '"statusurl":".*"' | cut -f4 -d\" | tr -d '\' 2>/dev/null`
if [ -z "$LOAD_STATUS_URL" ]
then
echo "No response from the REST API call."
echo "No response from the load product REST API call." >> report.txt
exit -1
fi

Expand Down Expand Up @@ -239,7 +239,7 @@ if [ $? -gt 0 ];then exit -1;fi
EXPORT_STATUS_URL=`echo $RESP | grep -o '"statusurl":".*"' | cut -f4 -d\" | tr -d '\' 2>/dev/null`
if [ -z "$EXPORT_STATUS_URL" ]
then
echo "No response from the REST API call."
echo "No response from the export REST API call." >> report.txt
exit -1
fi

Expand All @@ -266,7 +266,7 @@ then
# Can be 100% but still running
if [ "$STATUS" != "complete" ] && [ "$STATUS" != "running" ]
then
echo "Status of generation of Export JCL failed."
echo "Status of generation of Export JCL failed." >> report.txt
exit -1
fi
fi
Expand All @@ -275,7 +275,7 @@ done

if [ -z "$DSN" ]
then
echo "The creation of export JCL failed"
echo "The creation of export JCL failed" >> report.txt
exit -1
fi

Expand Down
28 changes: 28 additions & 0 deletions pswi/051_test_workflows.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/bin/sh
#version=1.0

export BASE_URL="${ZOSMF_URL}:${ZOSMF_PORT}"

echo ""
echo ""
echo "Script for testing workflows..."
echo "Host :" $ZOSMF_URL
echo "Port :" $ZOSMF_PORT
echo "SSH Port :" $ZZOW_SSH_PORT
echo "z/OSMF system :" $ZOSMF_SYSTEM
echo "Test HLQ :" $TEST_HLQ
echo "Test mount point :" $TEST_MOUNT
echo "Work zFS :" $WORK_ZFS # For z/OSMF v2.3
echo "Work mount point :" $WORK_MOUNT # For z/OSMF v2.3

echo "Mounting ${TEST_HLQ}.ZFS"
sh scripts/tmp_mounts.sh "${TEST_HLQ}.ZFS" "${TEST_MOUNT}"
if [ $? -gt 0 ]; then exit -1; fi

echo "Registering/testing the configuration workflow ${TEST_HLQ}.WORKFLOW(ZWECONF)"
sh scripts/wf_run_test.sh "${TEST_HLQ}.WORKFLOW(ZWECONF)"
if [ $? -gt 0 ];then exit -1;fi

echo "Registering/testing the configuration workflow ${TEST_MOUNT}/files/workflows/ZWECONF.xml"
sh scripts/wf_run_test.sh "${TEST_MOUNT}/files/workflows/ZWECONF.xml"
if [ $? -gt 0 ];then exit -1;fi
12 changes: 0 additions & 12 deletions pswi/05_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -72,15 +72,3 @@ echo " Running the deployment test for z/OSMF version 2.3"

pip install requests
python scripts/deploy_test_2_3.py

echo "Mounting ${TEST_HLQ}.ZFS"
sh scripts/tmp_mounts.sh "${TEST_HLQ}.ZFS" "${TEST_MOUNT}"
if [ $? -gt 0 ]; then exit -1; fi

echo "Registering/testing the configuration workflow ${TEST_HLQ}.WORKFLOW(ZWECONF)"
sh scripts/wf_run_test.sh "${TEST_HLQ}.WORKFLOW(ZWECONF)"
if [ $? -gt 0 ];then exit -1;fi

echo "Registering/testing the configuration workflow ${TEST_MOUNT}/content/files/workflows/ZWECONF.xml"
sh scripts/wf_run_test.sh "${TEST_MOUNT}/files/workflows/ZWECONF.xml"
if [ $? -gt 0 ];then exit -1;fi
50 changes: 35 additions & 15 deletions pswi/PSWI-marist.sh
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
set -x
export ZOSMF_URL="https://zzow07.zowe.marist.cloud"
export ZOSMF_PORT=10443
export ZOSMF_SYSTEM="S0W1"
export DIR="/u/zowead2"
export SMPEHLQ="ZOWEAD2"
export TMP_ZFS="ZOWEAD2.TMP.ZFS"
export DIR="/u/${ZOSMF_USER,,}"
export SMPEHLQ="${ZOSMF_USER}"
export TMP_ZFS="${ZOSMF_USER}.TMP.ZFS"
export ZOWE_MOUNT="/u/zwe/zowe-smpe/"
export VOLUME="ZOS003"
export TEST_HLQ="ZOWEAD2.PSWIT"
export TEST_HLQ="${ZOSMF_USER}.PSWIT"
export SYSAFF="(S0W1)"
export ACCOUNT=1

Expand All @@ -29,7 +30,7 @@ export TMP_MOUNT="${DIR}/zowe-tmp"
export TEST_MOUNT="${DIR}/test_mount"
export EXPORT="${TMP_MOUNT}/export/"
export WORK_MOUNT="${DIR}/work"
export WORK_ZFS="ZOWEAD2.WORK.ZFS"
export WORK_ZFS="${ZOSMF_USER}.WORK.ZFS"
export ZOSMF_V="2.3"
export SMPE_WF_NAME="ZOWE_SMPE_WF"
export PTF_WF_NAME="ZOWE_PTF_WF"
Expand All @@ -41,7 +42,7 @@ if [ -f ../.pax/zowe-smpe.zip ]; then
mkdir -p "unzipped"
unzip ../.pax/zowe-smpe.zip -d unzipped
else
echo "zowe-smpe file not found"
echo "zowe-smpe file not found" >> report.txt
exit -1
fi

Expand Down Expand Up @@ -73,7 +74,7 @@ else
if [ -f ../.pax/${FMID}.zip ]; then
unzip ../.pax/${FMID}.zip -d unzipped
else
echo "File with FMID not found"
echo "File with FMID not found" >> report.txt
exit -1
fi
fi
Expand All @@ -90,6 +91,14 @@ export WORKFLOW_DSN=${CSIHLQ}.WORKFLOW
export ZOWE_ZFS="${CSIHLQ}.ZFS"
export VERSION=$(cat ../manifest.json.template | grep -o '"version": ".*"' | head -1 | cut -f4 -d\")

# Initialize variables
presmpe=0
smpe=0
ptf=0
create=0
test=0
wf_test=0

# Upload and prepare all files
sh 00_presmpe.sh
presmpe=$?
Expand Down Expand Up @@ -125,7 +134,13 @@ if [ $presmpe -eq 0 ]; then
# Test PSWI
sh 05_test.sh
test=$?


if [ $test -eq 0 ]; then
#test the workflows
sh 051_test_workflows.sh
wf_test=$?
fi

# Cleanup after the test
sh 06_test_cleanup.sh
fi
Expand All @@ -141,23 +156,28 @@ if [ $presmpe -eq 0 ]; then
# Clean RELFILEs and PTFs
sh 08_presmpe_cleanup.sh
fi
else
# Clean RELFILEs and PTFs
sh 08_presmpe_cleanup.sh
fi

echo ""
echo ""

if [ $smpe -ne 0 ] || [ $ptf -ne 0 ] || [ $create -ne 0 ] || [ $test -ne 0 ] || [ $presmpe -ne 0 ]; then
echo "Build unsuccessful!"
if [ $smpe -ne 0 ] || [ $ptf -ne 0 ] || [ $create -ne 0 ] || [ $test -ne 0 ] || [ $presmpe -ne 0 ] || [ $wf_test -ne 0 ]; then
echo "Build unsuccessful!" >> report.txt
if [ $presmpe -ne 0 ]; then
echo "Pre-SMP/E wasn't successful."
echo "Pre-SMP/E wasn't successful." >> report.txt
elif [ $smpe -ne 0 ]; then
echo "SMP/E wasn't successful."
echo "SMP/E wasn't successful." >> report.txt
elif [ $ptf -ne 0 ]; then
echo "Applying PTFs wasn't successful."
echo "Applying PTFs wasn't successful." >> report.txt
elif [ $create -ne 0 ]; then
echo "Creation of PSWI wasn't successful."
echo "Creation of PSWI wasn't successful." >> report.txt
elif [ $test -ne 0 ]; then
echo "Testing of PSWI wasn't successful."
echo "Testing of PSWI wasn't successful." >> report.txt
elif [ $wf_test -ne 0 ]; then
echo "Workflow testing wasn't successful." >> report.txt
fi
exit -1
else
Expand Down
60 changes: 60 additions & 0 deletions pswi/ZWECONF_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
export ZOSMF_URL="https://zzow07.zowe.marist.cloud"
export ZOSMF_PORT=10443
export ZOSMF_SYSTEM="S0W1"
export JOBNAME="ZWECONF1"
export HOST=${ZOSMF_URL#https:\/\/}
export BASE_URL="${ZOSMF_URL}:${ZOSMF_PORT}"
WORK_MOUNT="/tmp"

echo "Changing runtime path in ZWECONF.properties."

cp ../workflows/files/ZWECONF.properties ./ZWECONF.properties
sed "s|runtimeDirectory=|runtimeDirectory=${WORK_MOUNT}|g" ./ZWECONF.properties > _ZWECONF
sed "s|java_home=|java_home=#delete_me#|g" _ZWECONF > ZWECONF
sed "s|node_home=|node_home=#delete_me#|g" ZWECONF > _ZWECONF

echo "Changing the configuration workflow to be fully automated."

cp ../workflows/files/ZWECONF.xml ./ZWECONF.xml
sed "s|<autoEnable>false|<autoEnable>true|g" ./ZWECONF.xml > ZWECONFX

sshpass -p${ZOSMF_PASS} sftp -o HostKeyAlgorithms=+ssh-rsa -o BatchMode=no -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -b - -P ${ZZOW_SSH_PORT} ${ZOSMF_USER}@${HOST} << EOF
cd ${WORK_MOUNT}
put _ZWECONF
put ZWECONFX
EOF

echo "Testing the configuration workflow ${WORK_MOUNT}/ZWECONFX"
sh scripts/wf_run_test.sh "${WORK_MOUNT}/ZWECONFX" "run" "ZWECONF" "${WORK_MOUNT}/_ZWECONF"
if [ $? -gt 0 ];then exit -1;fi

echo "Converting zowe.yaml"

echo "//${ZOSMF_SYSTEM} JOB (1),'PSWI',MSGCLASS=A,REGION=0M" > JCL
echo "/*JOBPARM SYSAFF=(${ZOSMF_SYSTEM})" >> JCL
echo "//UNPAXDIR EXEC PGM=BPXBATCH" >> JCL
echo "//STDOUT DD SYSOUT=*" >> JCL
echo "//STDERR DD SYSOUT=*" >> JCL
echo "//STDPARM DD *" >> JCL
echo "SH set -x;set -e;" >> JCL
echo "cd ${WORK_MOUNT};" >> JCL
echo "iconv -f IBM-1047 -t ISO8859-1 zowe.yaml > zowe_.yaml;" >> JCL
echo "/*" >> JCL

sh scripts/submit_jcl.sh "`cat JCL`"
if [ $? -gt 0 ];then exit -1;fi
rm JCL

sshpass -p${ZOSMF_PASS} sftp -o HostKeyAlgorithms=+ssh-rsa -o BatchMode=no -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -b - -P ${ZZOW_SSH_PORT} ${ZOSMF_USER}@${HOST} << EOF
cd ${WORK_MOUNT}
get zowe_.yaml
rm zowe_.yaml
EOF

pwd

cp ../example-zowe.yaml example-zowe.yaml

diff --ed example-zowe.yaml zowe_.yaml > diff.txt || true

diff --ed diff.txt scripts/base_diff.txt > final_diff.txt
Loading

0 comments on commit c88c59a

Please sign in to comment.