diff --git a/compute_endpoint/packaging/JenkinsFile b/compute_endpoint/packaging/JenkinsFile new file mode 100644 index 000000000..619341b4c --- /dev/null +++ b/compute_endpoint/packaging/JenkinsFile @@ -0,0 +1,88 @@ +#! groovy + +@Library('gcs-build-scripts') _ + +pipeline { + agent none + options { + buildDiscarder( + logRotator( + numToKeepStr: '5', + artifactNumToKeepStr: '1', + artifactDaysToKeepStr: '7' + ) + ) + timeout(time: 3, unit: 'HOURS') + disableConcurrentBuilds() + } + stages { + stage ("Prep source") { + agent {label "package_creator"} + steps { + checkout scm + script { + def py_full_version = sh(script: "/opt/globus-python/bin/python3 -c 'import sys; print(\"{}.{}.{}\".format(*sys.version_info))'", returnStdout: true).trim() + def (py_epoch_version, py_major_version, py_minor_version) = py_full_version.tokenize(".") + + def venv_py = "${env.WORKSPACE}/compute_endpoint/packaging/venv-py${py_epoch_version}${py_major_version}/bin/python" + sh "git clean -fdx" + sh "git checkout kevin_package_test" +// dir("compute_endpoint/") { +// env.PKG_TARBALL = "globus_compute_endpoint-${env.PKG_VERSION.replace("~", "")}.tar.gz" +// env.PREREQS_TARBALL = "globus_compute_endpoint-prereqs-py${env.PY_MAJOR_VERSION}-${env.PKG_VERSION}.tar.xz" +// } + + dir("compute_endpoint/packaging/") { + env.DEB_SOURCE_STASH_NAME = "${UUID.randomUUID()}"; +// env.RPM_SOURCE_STASH_NAME = "${UUID.randomUUID()}"; + sh(script: "make show_vars") + sh(script: "make setup_dist_for_deb setup_dist_for_rpm") + sh(script: "make show_vars") + echo "PY_MAJOR_VERSION = ${env.PY_MAJOR_VERSION}" + } + dir("compute_endpoint/") { + def pkg_version = sh(script: "${venv_py} setup.py --version", returnStdout: true).trim().replace("-", "~") + echo "PKG_VERSION = ${env.PKG_VERSION}" + } + env.PKG_TARBALL = "globus_compute_endpoint-${pkg_version.replace("~", "")}.tar.gz" + env.PREREQS_TARBALL = "globus_compute_endpoint-prereqs-py${py_epoch_version}-${pkg_version}.tar.xz" + dir("compute_endpoint/packaging/dist/") { + sh "ls -l ./" + sh "cp ${env.PKG_TARBALL} ${env.PREREQS_TARBALL} debbuild/globus-compute-agent/" + sh "ls -lR debbuild/globus-compute-agent/" + sh "mv debbuild/globus-compute-agent/ ${env.DEB_SOURCE_STASH_NAME}" +// sh "mv dist/rpmbuild/globus-compute-agent/ ${env.DEB_SOURCE_STASH_NAME}" + stash(name: env.DEB_SOURCE_STASH_NAME, includes: "${env.DEB_SOURCE_STASH_NAME}/**/*") +// stash(name: env.RPM_SOURCE_STASH_NAME, includes: "${env.RPM_SOURCE_STASH_NAME}/**/*") + } + } + } + } + stage ("Build packages") { + steps { + script { + lock(resource: 'globus-compute-agent-build') { + parallel "debian": { + def extra_tarball_map = [ + prereqs: env.PREREQS_TARBALL + ] + echo "extra_tarball_map = ${extra_tarball_map}" + env.DEB_ARTIFACTS_STASH = buildDebian( + env.DEB_SOURCE_STASH_NAME, + env.PKG_TARBALL, + require_gcs5_repo: true, + epic: null, + exclude: null, + extra_tarball_map: extra_tarball_map) +// }, "rpm": { +// env.RPM_ARTIFACTS_STASH = buildMock( +// env.RPM_SOURCE_STASH_NAME, +// env.PKG_TARBALL, +// true, +// null) + }, "failFast": true + } + } + } + } } +} diff --git a/compute_endpoint/packaging/Makefile b/compute_endpoint/packaging/Makefile index ff842d4ab..84dd6c05f 100644 --- a/compute_endpoint/packaging/Makefile +++ b/compute_endpoint/packaging/Makefile @@ -27,6 +27,7 @@ PIP_NAME_U := $(shell echo $(PIP_NAME_D) | tr '-' '_') PKG_VERSION := $(shell cd ../; "$(VENV_PY)" setup.py --version | tr '-' '~') PKG_WHEEL = $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl +PKG_TARBALL = $(PIP_NAME_U)-$(PKG_VERSION).tar.gz PREREQS_TARBALL_NAME = $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION).tar.xz OS_CODENAME := $(shell test -f /etc/os-release && . /etc/os-release; echo $${VERSION_CODENAME:-focal}) @@ -107,7 +108,7 @@ $(PKG_WHEEL): $(VENV_PY) { /bin/echo -e "\nBUILD COMPUTE FROM A RELEASE TAG (current branch: $$(git branch --show-current))"; exit 1; } \ fi \ && rm -rf tests/ \ - && "$(VENV_PY)" -m build --wheel -o ../../ \ + && "$(VENV_PY)" -m build -o ../../ \ ) wheel: $(PKG_WHEEL) ##-Make the wheel (note that this does *not* include dependencies) @@ -120,13 +121,9 @@ prereq_tarball: $(PREREQS_TARBALL_NAME) ##-Make a tarball of wheel dependencies dist: $(PREREQS_TARBALL_NAME) ##-Make the dist/ directory with prereqs and wheel ready for packaging step rm -rf dist/ \ && mkdir dist/ \ - && cp $(PREREQS_TARBALL_NAME) $(PKG_WHEEL) dist/ + && mv $(PREREQS_TARBALL_NAME) $(PKG_WHEEL) $(PKG_TARBALL) dist/ -deb_build_needs: ##-Check that necessary executables are available before starting the DEB build. - @[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; } - @dpkg-checkbuilddeps - -deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpoint (.deb) +setup_dist_for_deb: dist ##-Place Debian-build required files in dist/ ( cd dist/ \ && rm -rf debbuild/ \ && mkdir -p debbuild/$(PKG_NAME)/wheels/ \ @@ -148,17 +145,19 @@ deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpo -e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \ -e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \ -e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \ - && dpkg-buildpackage -uc -us \ ) + +deb_build_needs: ##-Check that necessary executables are available before starting the DEB build. + @[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; } + @dpkg-checkbuilddeps + +deb: deb_build_needs dist setup_dist_for_deb ##-Build a Debian package of the Globus Compute Endpoint (.deb) + (cd dist/debbuild/$(PKG_NAME)/; dpkg-buildpackage -uc -us) @echo "\nDEB package successfully built:" @ls -lh dist/debbuild/*deb -rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build. - @[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; } - -rpm: rpm_build_needs dist ##-Build an RPM package of the Globus Compute Endpoint (.rpm) +setup_dist_for_rpm: dist ##-Place RPM-build required files in dist/ (separated for CI building purposes) ( cd dist/ \ - && pwd && ls \ && rm -rf rpmbuild/ \ && mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SPECS,SRPMS} rpmbuild/SOURCES/$(PKG_NAME)/wheels \ && tar -C rpmbuild/SOURCES/$(PKG_NAME)/wheels -xf "$(PREREQS_TARBALL_NAME)" \ @@ -170,7 +169,12 @@ rpm: rpm_build_needs dist ##-Build an RPM package of the Globus Compute Endpoin -e "s/@PACKAGE_WHEEL@/$(PKG_WHEEL)/g" \ -e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \ < ../fedora/$(PKG_NAME).spec.in > ./$(PKG_NAME).spec \ - && HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec \ ) + +rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build. + @[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; } + +rpm: rpm_build_needs dist setup_dist_for_rpm ##-Build an RPM package of the Globus Compute Endpoint (.rpm) + (cd dist/; HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec) @echo -e "\nRPM package successfully built:" @ls -lh dist/rpmbuild/RPMS/**/*rpm