Skip to content

Commit

Permalink
Jenkins WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
khk-globus committed Apr 26, 2024
1 parent e5e2212 commit 5dafa37
Show file tree
Hide file tree
Showing 5 changed files with 146 additions and 30 deletions.
87 changes: 87 additions & 0 deletions compute_endpoint/packaging/JenkinsFile
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
#! groovy

@Library('gcs-build-scripts') _

pipeline {
agent none
options {
buildDiscarder(
logRotator(
numToKeepStr: '5',
artifactNumToKeepStr: '1',
artifactDaysToKeepStr: '7'
)
)
timeout(time: 3, unit: 'HOURS')
disableConcurrentBuilds()
}
stages {
stage ("Prep source") {
agent {label "package_creator"}
steps {
checkout scm
script {
env.DEB_SOURCE_STASH_NAME = "${UUID.randomUUID()}";
env.RPM_SOURCE_STASH_NAME = "${UUID.randomUUID()}";
def py_full_version = sh(script: "/opt/globus-python/bin/python3 -c 'import sys; print(\"{}.{}.{}\".format(*sys.version_info))'", returnStdout: true).trim()
def (py_epoch_version, py_major_version, py_minor_version) = py_full_version.tokenize(".")
def pkg_version = "not yet set"

def venv_py = "${env.WORKSPACE}/compute_endpoint/packaging/venv-${py_epoch_version}${py_major_version}/bin/python"
sh "git clean -fdx"
sh "git checkout kevin_package_test"

dir("compute_endpoint/packaging/") {
sh(script: "make show_vars")
// sh(script: "make setup_dist_for_deb setup_dist_for_rpm")
sh(script: "make setup_dist_for_rpm")
sh(script: "make show_vars")
}
dir("compute_endpoint/") {
pkg_version = sh(script: "${venv_py} setup.py --version", returnStdout: true).trim().replace("-", "~")
echo "pkg_version = ${pkg_version}"
}
env.PKG_TARBALL = "globus_compute_endpoint-${pkg_version.replace("~", "")}.tar.gz"
env.PREREQS_TARBALL = "globus_compute_endpoint-prereqs-py${py_epoch_version}${py_major_version}-${pkg_version}.tar.gz"
dir("compute_endpoint/packaging/dist/") {
sh "ls -l ./"
sh "cp ${env.PKG_TARBALL} ${env.PREREQS_TARBALL} debbuild/globus-compute-agent/"
sh "ls -lR debbuild/globus-compute-agent/"
sh "mv debbuild/globus-compute-agent/ ${env.DEB_SOURCE_STASH_NAME}"
sh "mv dist/rpmbuild/ ${env.RPM_SOURCE_STASH_NAME}"
stash(name: env.DEB_SOURCE_STASH_NAME, includes: "${env.DEB_SOURCE_STASH_NAME}/**/*")
stash(name: env.RPM_SOURCE_STASH_NAME, includes: "${env.RPM_SOURCE_STASH_NAME}/**/*")
}
}
}
}
stage ("Build packages") {
steps {
script {
lock(resource: 'globus-compute-agent-build') {
// parallel "debian": {
parallel "rpm": {
def extra_tarball_map = [
prereqs: env.PREREQS_TARBALL
]
echo "extra_tarball_map = ${extra_tarball_map}"
// env.DEB_ARTIFACTS_STASH = buildDebian(
// env.DEB_SOURCE_STASH_NAME,
// env.PKG_TARBALL,
// require_gcs5_repo: true,
// epic: null,
// exclude: null,
// extra_tarball_map: extra_tarball_map)
// }, "rpm": {

env.RPM_ARTIFACTS_STASH = buildMock(
env.RPM_SOURCE_STASH_NAME,
env.PKG_TARBALL,
true,
null)
}, "failFast": true
}
}
}
} }
}
69 changes: 46 additions & 23 deletions compute_endpoint/packaging/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ PIP_NAME_U := $(shell echo $(PIP_NAME_D) | tr '-' '_')

PKG_VERSION := $(shell cd ../; "$(VENV_PY)" setup.py --version | tr '-' '~')
PKG_WHEEL = $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl
PREREQS_TARBALL_NAME = $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION).tar.xz
PKG_TARBALL = $(PIP_NAME_U)-$(PKG_VERSION).tar.gz
PREREQS_TARBALL_NAME = $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION).tar.gz

OS_CODENAME := $(shell test -f /etc/os-release && . /etc/os-release; echo $${VERSION_CODENAME:-focal})

Expand Down Expand Up @@ -63,6 +64,7 @@ show_vars: ##-For debugging, show the Makefile variables; will install a venv
@echo "PKG_NAME : $(PKG_NAME)"
@echo "PKG_VERSION : $(PKG_VERSION)"
@echo "PREREQS_TARBALL_NAME : $(PREREQS_TARBALL_NAME)"
@echo "PKG_TARBALL : $(PKG_TARBALL)"
@echo "PKG_WHEEL : $(PKG_WHEEL)"
@echo
@echo " Override python path with PYTHON3 variable:"
Expand Down Expand Up @@ -107,7 +109,7 @@ $(PKG_WHEEL): $(VENV_PY)
{ /bin/echo -e "\nBUILD COMPUTE FROM A RELEASE TAG (current branch: $$(git branch --show-current))"; exit 1; } \
fi \
&& rm -rf tests/ \
&& "$(VENV_PY)" -m build --wheel -o ../../ \
&& "$(VENV_PY)" -m build -o ../../ \
)

wheel: $(PKG_WHEEL) ##-Make the wheel (note that this does *not* include dependencies)
Expand All @@ -117,21 +119,35 @@ $(PREREQS_TARBALL_NAME): $(VENV_PY) $(PKG_WHEEL)

prereq_tarball: $(PREREQS_TARBALL_NAME) ##-Make a tarball of wheel dependencies

dist: $(PREREQS_TARBALL_NAME) ##-Make the dist/ directory with prereqs and wheel ready for packaging step
rm -rf dist/ \
&& mkdir dist/ \
&& cp $(PREREQS_TARBALL_NAME) $(PKG_WHEEL) dist/
_dist:
rm -rf dist/ && mkdir dist/
cp "$(PKG_WHEEL)" dist/

# For CI, which does things differently than this Makefile:
# - append PKG_WHEEL to *prereqs* tarball
# - append package_shim.sh to *package* tarball
( _prereq_dir="$$(tar -tf "$(PREREQS_TARBALL_NAME)" | head -1)" \
&& _pkg_dir="$$(tar -tf "$(PKG_TARBALL)" | head -1)" \
&& cd dist/ \
&& tar -xf "../$(PREREQS_TARBALL_NAME)" \
&& cp "$(PKG_WHEEL)" "$${_prereq_dir}" \
&& tar -czf "$(PREREQS_TARBALL_NAME)" "$${_prereq_dir}" \
&& rm -r "$${_prereq_dir}" \
&& tar -xf "../$(PKG_TARBALL)" \
&& cp "../package_shim.sh" "$${_pkg_dir}" \
&& tar -czf "$(PKG_TARBALL)" "$${_pkg_dir}" \
&& rm -r "$${_pkg_dir}" \
)

deb_build_needs: ##-Check that necessary executables are available before starting the DEB build.
@[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; }
@dpkg-checkbuilddeps
dist: $(PREREQS_TARBALL_NAME) _dist ##-Make the dist/ directory with prereqs and wheel ready for packaging step
rm "$(PKG_WHEEL)" "$(PKG_TARBALL)" "$(PREREQS_TARBALL_NAME)"

deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpoint (.deb)
setup_dist_for_deb: dist ##-Place Debian-build required files in dist/
( cd dist/ \
&& rm -rf debbuild/ \
&& mkdir -p debbuild/$(PKG_NAME)/wheels/ \
&& tar -C debbuild/$(PKG_NAME)/wheels/ -xf "$(PREREQS_TARBALL_NAME)" \
&& cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/wheels/ \
&& mkdir -p debbuild/$(PKG_NAME)/prereqs/ \
&& tar -C debbuild/$(PKG_NAME)/prereqs/ --strip 1 -xf "$(PREREQS_TARBALL_NAME)" \
&& cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/prereqs/ \
&& cp -R ../debian debbuild/$(PKG_NAME)/ \
&& cp ../package_shim.sh debbuild/$(PKG_NAME)/ \
&& cd debbuild/$(PKG_NAME)/ \
Expand All @@ -148,29 +164,36 @@ deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpo
-e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \
-e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \
-e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \
&& dpkg-buildpackage -uc -us \
)

deb_build_needs: ##-Check that necessary executables are available before starting the DEB build.
@[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; }
@dpkg-checkbuilddeps

deb: deb_build_needs dist setup_dist_for_deb ##-Build a Debian package of the Globus Compute Endpoint (.deb)
(cd dist/debbuild/$(PKG_NAME)/; dpkg-buildpackage -uc -us)
@echo "\nDEB package successfully built:"
@ls -lh dist/debbuild/*deb

rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build.
@[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; }

rpm: rpm_build_needs dist ##-Build an RPM package of the Globus Compute Endpoint (.rpm)
setup_dist_for_rpm: dist ##-Place RPM-build required files in dist/ (separated for CI building purposes)
( cd dist/ \
&& pwd && ls \
&& rm -rf rpmbuild/ \
&& mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SPECS,SRPMS} rpmbuild/SOURCES/$(PKG_NAME)/wheels \
&& tar -C rpmbuild/SOURCES/$(PKG_NAME)/wheels -xf "$(PREREQS_TARBALL_NAME)" \
&& cp $(PKG_WHEEL) rpmbuild/SOURCES/$(PKG_NAME)/wheels/ \
&& mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SPECS,SRPMS} rpmbuild/SOURCES/$(PKG_NAME)/prereqs/ \
&& tar -C rpmbuild/SOURCES/$(PKG_NAME)/prereqs/ --strip 1 -xf "$(PREREQS_TARBALL_NAME)" \
&& cp $(PKG_WHEEL) rpmbuild/SOURCES/$(PKG_NAME)/prereqs/ \
&& cp ../package_shim.sh rpmbuild/SOURCES/$(PKG_NAME)/ \
&& sed \
-e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \
-e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \
-e "s/@PACKAGE_WHEEL@/$(PKG_WHEEL)/g" \
-e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \
< ../fedora/$(PKG_NAME).spec.in > ./$(PKG_NAME).spec \
&& HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec \
)

rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build.
@[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; }

rpm: rpm_build_needs dist setup_dist_for_rpm ##-Build an RPM package of the Globus Compute Endpoint (.rpm)
(cd dist/; HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec)
@echo -e "\nRPM package successfully built:"
@ls -lh dist/rpmbuild/RPMS/**/*rpm
6 changes: 3 additions & 3 deletions compute_endpoint/packaging/create-prereqs-tarball.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,12 @@ done > dependent-prereqs.txt

modified_time="$(TZ=UTC0 date --rfc-3339=seconds)"

TZ=UTC0 tar -C "$download_dir/" --format=posix \
TZ=UTC0 tar --format=posix \
--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime,delete=mtime \
--mtime="$modified_time" \
--numeric-owner \
--owner=0 \
--group=0 \
--mode="go-rwx,u+rw" \
-cf - . \
| xz -9
-cf - "$download_dir/" \
| gzip -9
2 changes: 1 addition & 1 deletion compute_endpoint/packaging/debian/control
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Source: @PACKAGE_NAME@
Priority: optional
Section: contrib/science
Maintainer: Globus Support <[email protected]>
Build-Depends: debhelper (>= 13.6), globus-python (>= 3.9), dh-exec, dh-python
Build-Depends: debhelper (>= 12), globus-python (>= 3.9), dh-exec, dh-python
Standards-Version: 4.6.0
Homepage: https://globus-compute.readthedocs.io/

Expand Down
12 changes: 9 additions & 3 deletions compute_endpoint/packaging/debian/rules
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,13 @@ _unitdir=/lib/systemd/system
.PHONY: override_dh_auto_configure
override_dh_auto_configure:
$(PYTHON3) -mvenv $${PWD}/$(TMP_VIRTUAL_ENV)
. "$(TMP_VIRTUAL_ENV)/bin/activate"; \
. "$${PWD}/$(TMP_VIRTUAL_ENV)/bin/activate"; \
set -x; \
python3 -mpip install --no-index --no-cache-dir -I --compile -U wheels/pip-*.whl; \
python3 -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/wheels/ "@PIP_NAME@"
pwd; \
ls -lR; \
pwd; \
python -mpip install --no-index --no-cache-dir -I --compile -U prereqs/pip-*.whl; \
python -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/prereqs/ "@PIP_NAME@"

.PHONY: override_dh_auto_clean
override_dh_auto_clean:
Expand Down Expand Up @@ -78,5 +81,8 @@ override_dh_python3:
override_dh_builddeb:
dh_builddeb -- -Zxz

.PHONY: override_dh_auto_build
override_dh_auto_build: # empty == we do it ourselves in auto_install (above)

%:
dh $@ --with python3 --with systemd

0 comments on commit 5dafa37

Please sign in to comment.